From 327d857b6e9b8517a51253d2caa88311a3e27068 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 7 Mar 2024 19:52:48 +0530 Subject: [PATCH 01/13] get nodes in batches in dependency flow --- .../java/com/akto/parsers/HttpCallParser.java | 6 +- .../dependency/TestDependencyAnalyser.java | 12 +- .../src/main/java/com/akto/utils/Utils.java | 15 +- .../dto/dependency_flow/DependencyFlow.java | 191 +++++++++++------- 4 files changed, 137 insertions(+), 87 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 42f0603b51..e7d5ed0de5 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -172,9 +172,9 @@ public void syncFunction(List responseParams, boolean syncIm apiCatalogSync.computeDelta(aggregator, false, apiCollectionId); } - // for (HttpResponseParams responseParam: filteredResponseParams) { - // dependencyAnalyser.analyse(responseParam.getOrig(), responseParam.requestParams.getApiCollectionId()); - // } + for (HttpResponseParams responseParam: filteredResponseParams) { + dependencyAnalyser.analyse(responseParam.getOrig(), responseParam.requestParams.getApiCollectionId()); + } this.sync_count += filteredResponseParams.size(); int syncThresh = numberOfSyncs < 10 ? 10000 : sync_threshold_count; diff --git a/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java b/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java index a6264156a6..743ab78023 100644 --- a/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java +++ b/apps/api-runtime/src/test/java/com/akto/dependency/TestDependencyAnalyser.java @@ -59,7 +59,7 @@ public static void process(List messages, int expectedNodes) { } } - // @Test + @Test public void testAnalyse1() { // initial node String message1 = "{\"method\":\"POST\",\"requestPayload\":\"{'m1_req_key_a': 'm1_req_val_a', 'm1_req_key_b': 'm1_req_val_b'}\",\"responsePayload\":\"{'m1_resp_key_a': 'm1_resp_val_a', 'm1_resp_key_b': 'm1_resp_val_b', 'm1_resp_key_c': 'm1_resp_val_c'}\",\"ip\":\"null\",\"source\":\"HAR\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":\"1000\",\"path\":\"/api/m1\",\"requestHeaders\":\"{\\\"Cookie\\\":\\\"JSESSIONID=node0e7rms6jdk2u41w0drvjv8hkoo0.node0; mp_c403d0b00353cc31d7e33d68dc778806_mixpanel=%7B%22distinct_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24device_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24initial_referrer%22%3A%20%22%24direct%22%2C%22%24initial_referring_domain%22%3A%20%22%24direct%22%7D\\\",\\\"Origin\\\":\\\"dev-1.akto.io\\\",\\\"Accept\\\":\\\"application/json, text/plain, */*\\\",\\\"Access-Control-Allow-Origin\\\":\\\"*\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Referer\\\":\\\"https://dev-1.akto.io/dashboard/settings\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"dev-1.akto.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"access-token\\\":\\\"eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoibG9naW4iLCJzaWduZWRVcCI6InRydWUiLCJ1c2VybmFtZSI6InNoaXZhbnNoQGFrdG8uaW8iLCJpYXQiOjE2NjE4ODAwNTYsImV4cCI6MTY2MTg4MDk1Nn0.wxDbUhIfhX6i8tITykZcdztg8CZUcrBvdqbLgiZJN0Q4QkGOvhHozZ6lwgFzQe3hTOxuFOv8wxg4E_vzruLMgSRmapHGuTi57qTYFWIJNb-VSUa_Nz__t6aXOaXYckO2nvzN2rp1qeTIEKrhLaC_nV5gZpOB2fnBC2Yr1KasERpdDO7I0xc4dqdLQXQRxrWgP6lKlkGKHziCrkvLEWqC7mXrRsS23m-qv4pELm0MikIqf-fl4wmwj7g42769APwAuoQdIgMnUOx2rT1ewkcW72py3wveX96oomdDyvIM6_y5uYALsTymc0xxr1yZOT9Gseypbjm-sa7byVaSbw2s9g\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Chromium\\\\\\\";v=\\\\\\\"104\\\\\\\", \\\\\\\" Not A;Brand\\\\\\\";v=\\\\\\\"99\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"104\\\\\\\"\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"Windows\\\\\\\"\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.9\\\",\\\"Content-Length\\\":\\\"2\\\",\\\"account\\\":\\\"1000000\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"X-Frame-Options\\\":\\\"deny\\\",\\\"Cache-Control\\\":\\\"no-cache, no-store, must-revalidate, pre-check=0, post-check=0\\\",\\\"Server\\\":\\\"AKTO server\\\",\\\"X-Content-Type-Options\\\":\\\"nosniff\\\",\\\"Content-Encoding\\\":\\\"gzip\\\",\\\"Vary\\\":\\\"Accept-Encoding, User-Agent\\\",\\\"Content-Length\\\":\\\"150\\\",\\\"X-XSS-Protection\\\":\\\"1\\\",\\\"Content-Language\\\":\\\"en-US\\\",\\\"Date\\\":\\\"Tue, 30 Aug 2022 17:22:40 GMT\\\",\\\"Content-Type\\\":\\\"application/json;charset=utf-8\\\"}\",\"time\":\"1661880160\",\"contentType\":\"application/json;charset=utf-8\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; @@ -103,7 +103,7 @@ public void testAnalyse1() { assertEquals(1, connections.size()); // even though 2 apis are giving data but same param } - // @Test + @Test public void testAnalyse2() { String message1 = "{\"method\":\"POST\",\"requestPayload\":\"{}\",\"responsePayload\":\"{'user': {'name': 'user1', 'age': 0}}\",\"ip\":\"null\",\"source\":\"HAR\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":\"1000\",\"path\":\"/api/user\",\"requestHeaders\":\"{\\\"Cookie\\\":\\\"JSESSIONID=node0e7rms6jdk2u41w0drvjv8hkoo0.node0; mp_c403d0b00353cc31d7e33d68dc778806_mixpanel=%7B%22distinct_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24device_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24initial_referrer%22%3A%20%22%24direct%22%2C%22%24initial_referring_domain%22%3A%20%22%24direct%22%7D\\\",\\\"Origin\\\":\\\"dev-1.akto.io\\\",\\\"Accept\\\":\\\"application/json, text/plain, */*\\\",\\\"Access-Control-Allow-Origin\\\":\\\"*\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Referer\\\":\\\"https://dev-1.akto.io/dashboard/settings\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"dev-1.akto.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"access-token\\\":\\\"eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoibG9naW4iLCJzaWduZWRVcCI6InRydWUiLCJ1c2VybmFtZSI6InNoaXZhbnNoQGFrdG8uaW8iLCJpYXQiOjE2NjE4ODAwNTYsImV4cCI6MTY2MTg4MDk1Nn0.wxDbUhIfhX6i8tITykZcdztg8CZUcrBvdqbLgiZJN0Q4QkGOvhHozZ6lwgFzQe3hTOxuFOv8wxg4E_vzruLMgSRmapHGuTi57qTYFWIJNb-VSUa_Nz__t6aXOaXYckO2nvzN2rp1qeTIEKrhLaC_nV5gZpOB2fnBC2Yr1KasERpdDO7I0xc4dqdLQXQRxrWgP6lKlkGKHziCrkvLEWqC7mXrRsS23m-qv4pELm0MikIqf-fl4wmwj7g42769APwAuoQdIgMnUOx2rT1ewkcW72py3wveX96oomdDyvIM6_y5uYALsTymc0xxr1yZOT9Gseypbjm-sa7byVaSbw2s9g\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Chromium\\\\\\\";v=\\\\\\\"104\\\\\\\", \\\\\\\" Not A;Brand\\\\\\\";v=\\\\\\\"99\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"104\\\\\\\"\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"Windows\\\\\\\"\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.9\\\",\\\"Content-Length\\\":\\\"2\\\",\\\"account\\\":\\\"1000000\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"X-Frame-Options\\\":\\\"deny\\\",\\\"Cache-Control\\\":\\\"no-cache, no-store, must-revalidate, pre-check=0, post-check=0\\\",\\\"Server\\\":\\\"AKTO server\\\",\\\"X-Content-Type-Options\\\":\\\"nosniff\\\",\\\"Content-Encoding\\\":\\\"gzip\\\",\\\"Vary\\\":\\\"Accept-Encoding, User-Agent\\\",\\\"Content-Length\\\":\\\"150\\\",\\\"X-XSS-Protection\\\":\\\"1\\\",\\\"Content-Language\\\":\\\"en-US\\\",\\\"Date\\\":\\\"Tue, 30 Aug 2022 17:22:40 GMT\\\",\\\"Content-Type\\\":\\\"application/json;charset=utf-8\\\"}\",\"time\":\"1661880160\",\"contentType\":\"application/json;charset=utf-8\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; String message2 = "{\"method\":\"POST\",\"requestPayload\":\"{'name': 'user1'}\",\"responsePayload\":\"{ 'company': 'akto.io' }\",\"ip\":\"null\",\"source\":\"HAR\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":\"1000\",\"path\":\"/api/user_info\",\"requestHeaders\":\"{\\\"Cookie\\\":\\\"JSESSIONID=node0e7rms6jdk2u41w0drvjv8hkoo0.node0; mp_c403d0b00353cc31d7e33d68dc778806_mixpanel=%7B%22distinct_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24device_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24initial_referrer%22%3A%20%22%24direct%22%2C%22%24initial_referring_domain%22%3A%20%22%24direct%22%7D\\\",\\\"Origin\\\":\\\"dev-1.akto.io\\\",\\\"Accept\\\":\\\"application/json, text/plain, */*\\\",\\\"Access-Control-Allow-Origin\\\":\\\"*\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Referer\\\":\\\"https://dev-1.akto.io/dashboard/settings\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"dev-1.akto.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"access-token\\\":\\\"eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoibG9naW4iLCJzaWduZWRVcCI6InRydWUiLCJ1c2VybmFtZSI6InNoaXZhbnNoQGFrdG8uaW8iLCJpYXQiOjE2NjE4ODAwNTYsImV4cCI6MTY2MTg4MDk1Nn0.wxDbUhIfhX6i8tITykZcdztg8CZUcrBvdqbLgiZJN0Q4QkGOvhHozZ6lwgFzQe3hTOxuFOv8wxg4E_vzruLMgSRmapHGuTi57qTYFWIJNb-VSUa_Nz__t6aXOaXYckO2nvzN2rp1qeTIEKrhLaC_nV5gZpOB2fnBC2Yr1KasERpdDO7I0xc4dqdLQXQRxrWgP6lKlkGKHziCrkvLEWqC7mXrRsS23m-qv4pELm0MikIqf-fl4wmwj7g42769APwAuoQdIgMnUOx2rT1ewkcW72py3wveX96oomdDyvIM6_y5uYALsTymc0xxr1yZOT9Gseypbjm-sa7byVaSbw2s9g\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Chromium\\\\\\\";v=\\\\\\\"104\\\\\\\", \\\\\\\" Not A;Brand\\\\\\\";v=\\\\\\\"99\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"104\\\\\\\"\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"Windows\\\\\\\"\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.9\\\",\\\"Content-Length\\\":\\\"2\\\",\\\"account\\\":\\\"1000000\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"X-Frame-Options\\\":\\\"deny\\\",\\\"Cache-Control\\\":\\\"no-cache, no-store, must-revalidate, pre-check=0, post-check=0\\\",\\\"Server\\\":\\\"AKTO server\\\",\\\"X-Content-Type-Options\\\":\\\"nosniff\\\",\\\"Content-Encoding\\\":\\\"gzip\\\",\\\"Vary\\\":\\\"Accept-Encoding, User-Agent\\\",\\\"Content-Length\\\":\\\"150\\\",\\\"X-XSS-Protection\\\":\\\"1\\\",\\\"Content-Language\\\":\\\"en-US\\\",\\\"Date\\\":\\\"Tue, 30 Aug 2022 17:22:40 GMT\\\",\\\"Content-Type\\\":\\\"application/json;charset=utf-8\\\"}\",\"time\":\"1661880160\",\"contentType\":\"application/json;charset=utf-8\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; @@ -126,7 +126,7 @@ public void testAnalyse2() { } - // @Test + @Test public void testAnalyse3() { // initial node String message1 = "{\"method\":\"POST\",\"requestPayload\":\"{'m1_req_key_a': 'm1_req_val_a', 'm1_req_key_b': 'm1_req_val_b'}\",\"responsePayload\":\"{'m1_resp_key_a': 'm1_resp_val_a', 'm1_resp_key_b': 'm1_resp_val_b', 'm1_resp_key_c': 'm1_resp_val_c'}\",\"ip\":\"null\",\"source\":\"HAR\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":\"1000\",\"path\":\"/api/m1\",\"requestHeaders\":\"{\\\"Cookie\\\":\\\"JSESSIONID=node0e7rms6jdk2u41w0drvjv8hkoo0.node0; mp_c403d0b00353cc31d7e33d68dc778806_mixpanel=%7B%22distinct_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24device_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24initial_referrer%22%3A%20%22%24direct%22%2C%22%24initial_referring_domain%22%3A%20%22%24direct%22%7D\\\",\\\"Origin\\\":\\\"dev-1.akto.io\\\",\\\"Accept\\\":\\\"application/json, text/plain, */*\\\",\\\"Access-Control-Allow-Origin\\\":\\\"*\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Referer\\\":\\\"https://dev-1.akto.io/dashboard/settings\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"dev-1.akto.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"access-token\\\":\\\"eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoibG9naW4iLCJzaWduZWRVcCI6InRydWUiLCJ1c2VybmFtZSI6InNoaXZhbnNoQGFrdG8uaW8iLCJpYXQiOjE2NjE4ODAwNTYsImV4cCI6MTY2MTg4MDk1Nn0.wxDbUhIfhX6i8tITykZcdztg8CZUcrBvdqbLgiZJN0Q4QkGOvhHozZ6lwgFzQe3hTOxuFOv8wxg4E_vzruLMgSRmapHGuTi57qTYFWIJNb-VSUa_Nz__t6aXOaXYckO2nvzN2rp1qeTIEKrhLaC_nV5gZpOB2fnBC2Yr1KasERpdDO7I0xc4dqdLQXQRxrWgP6lKlkGKHziCrkvLEWqC7mXrRsS23m-qv4pELm0MikIqf-fl4wmwj7g42769APwAuoQdIgMnUOx2rT1ewkcW72py3wveX96oomdDyvIM6_y5uYALsTymc0xxr1yZOT9Gseypbjm-sa7byVaSbw2s9g\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Chromium\\\\\\\";v=\\\\\\\"104\\\\\\\", \\\\\\\" Not A;Brand\\\\\\\";v=\\\\\\\"99\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"104\\\\\\\"\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"Windows\\\\\\\"\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.9\\\",\\\"Content-Length\\\":\\\"2\\\",\\\"account\\\":\\\"1000000\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"X-Frame-Options\\\":\\\"deny\\\",\\\"Cache-Control\\\":\\\"no-cache, no-store, must-revalidate, pre-check=0, post-check=0\\\",\\\"Server\\\":\\\"AKTO server\\\",\\\"X-Content-Type-Options\\\":\\\"nosniff\\\",\\\"Content-Encoding\\\":\\\"gzip\\\",\\\"Vary\\\":\\\"Accept-Encoding, User-Agent\\\",\\\"Content-Length\\\":\\\"150\\\",\\\"X-XSS-Protection\\\":\\\"1\\\",\\\"Content-Language\\\":\\\"en-US\\\",\\\"Date\\\":\\\"Tue, 30 Aug 2022 17:22:40 GMT\\\",\\\"Content-Type\\\":\\\"application/json;charset=utf-8\\\"}\",\"time\":\"1661880160\",\"contentType\":\"application/json;charset=utf-8\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; @@ -171,7 +171,7 @@ public void testAnalyse3() { } - // @Test + @Test public void testAnalyse4() { // initial node String message1 = "{\"method\":\"POST\",\"requestPayload\":\"{'m1_req_key_a': 'm1_req_val_a', 'm1_req_key_b': 'm1_req_val_b'}\",\"responsePayload\":\"{'m1_resp_key_a': 'm1_resp_val_a', 'm1_resp_key_b': 'm1_resp_val_b', 'm1_resp_key_c': 'm1_resp_val_c'}\",\"ip\":\"null\",\"source\":\"HAR\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":\"1000\",\"path\":\"/api/books/1\",\"requestHeaders\":\"{\\\"Cookie\\\":\\\"JSESSIONID=node0e7rms6jdk2u41w0drvjv8hkoo0.node0; mp_c403d0b00353cc31d7e33d68dc778806_mixpanel=%7B%22distinct_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24device_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24initial_referrer%22%3A%20%22%24direct%22%2C%22%24initial_referring_domain%22%3A%20%22%24direct%22%7D\\\",\\\"Origin\\\":\\\"dev-1.akto.io\\\",\\\"Accept\\\":\\\"application/json, text/plain, */*\\\",\\\"Access-Control-Allow-Origin\\\":\\\"*\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Referer\\\":\\\"https://dev-1.akto.io/dashboard/settings\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"dev-1.akto.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"access-token\\\":\\\"eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoibG9naW4iLCJzaWduZWRVcCI6InRydWUiLCJ1c2VybmFtZSI6InNoaXZhbnNoQGFrdG8uaW8iLCJpYXQiOjE2NjE4ODAwNTYsImV4cCI6MTY2MTg4MDk1Nn0.wxDbUhIfhX6i8tITykZcdztg8CZUcrBvdqbLgiZJN0Q4QkGOvhHozZ6lwgFzQe3hTOxuFOv8wxg4E_vzruLMgSRmapHGuTi57qTYFWIJNb-VSUa_Nz__t6aXOaXYckO2nvzN2rp1qeTIEKrhLaC_nV5gZpOB2fnBC2Yr1KasERpdDO7I0xc4dqdLQXQRxrWgP6lKlkGKHziCrkvLEWqC7mXrRsS23m-qv4pELm0MikIqf-fl4wmwj7g42769APwAuoQdIgMnUOx2rT1ewkcW72py3wveX96oomdDyvIM6_y5uYALsTymc0xxr1yZOT9Gseypbjm-sa7byVaSbw2s9g\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Chromium\\\\\\\";v=\\\\\\\"104\\\\\\\", \\\\\\\" Not A;Brand\\\\\\\";v=\\\\\\\"99\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"104\\\\\\\"\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"Windows\\\\\\\"\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.9\\\",\\\"Content-Length\\\":\\\"2\\\",\\\"account\\\":\\\"1000000\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"X-Frame-Options\\\":\\\"deny\\\",\\\"Cache-Control\\\":\\\"no-cache, no-store, must-revalidate, pre-check=0, post-check=0\\\",\\\"Server\\\":\\\"AKTO server\\\",\\\"X-Content-Type-Options\\\":\\\"nosniff\\\",\\\"Content-Encoding\\\":\\\"gzip\\\",\\\"Vary\\\":\\\"Accept-Encoding, User-Agent\\\",\\\"Content-Length\\\":\\\"150\\\",\\\"X-XSS-Protection\\\":\\\"1\\\",\\\"Content-Language\\\":\\\"en-US\\\",\\\"Date\\\":\\\"Tue, 30 Aug 2022 17:22:40 GMT\\\",\\\"Content-Type\\\":\\\"application/json;charset=utf-8\\\"}\",\"time\":\"1661880160\",\"contentType\":\"application/json;charset=utf-8\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; @@ -204,7 +204,7 @@ public void testAnalyse4() { } - // @Test + @Test public void testMergeNodes() { Map nodes = new HashMap<>(); int now = Context.now(); @@ -269,7 +269,7 @@ public void testMergeNodes() { assertEquals(4, dependencyNode3New.getParamInfos().get(0).getCount()); } - // @Test + @Test public void testUrlParameter() { // initial node String message1 = "{\"method\":\"POST\",\"requestPayload\":\"{'m1_req_key_a': 'm1_req_val_a', 'm1_req_key_b': 'm1_req_val_b'}\",\"responsePayload\":\"{'m1_resp_key_a': 'm1_resp_val_a', 'm1_resp_key_b': 'b9bc0f40-cbb9-45eb-ae7d-29dde8da2deb',}\",\"ip\":\"null\",\"source\":\"HAR\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":\"1000\",\"path\":\"/api/books\",\"requestHeaders\":\"{\\\"Cookie\\\":\\\"JSESSIONID=node0e7rms6jdk2u41w0drvjv8hkoo0.node0; mp_c403d0b00353cc31d7e33d68dc778806_mixpanel=%7B%22distinct_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24device_id%22%3A%20%22182edbc00381d9-063588c46d5c5e-26021d51-144000-182edbc0039615%22%2C%22%24initial_referrer%22%3A%20%22%24direct%22%2C%22%24initial_referring_domain%22%3A%20%22%24direct%22%7D\\\",\\\"Origin\\\":\\\"dev-1.akto.io\\\",\\\"Accept\\\":\\\"application/json, text/plain, */*\\\",\\\"Access-Control-Allow-Origin\\\":\\\"*\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Referer\\\":\\\"https://dev-1.akto.io/dashboard/settings\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"dev-1.akto.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"access-token\\\":\\\"eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoibG9naW4iLCJzaWduZWRVcCI6InRydWUiLCJ1c2VybmFtZSI6InNoaXZhbnNoQGFrdG8uaW8iLCJpYXQiOjE2NjE4ODAwNTYsImV4cCI6MTY2MTg4MDk1Nn0.wxDbUhIfhX6i8tITykZcdztg8CZUcrBvdqbLgiZJN0Q4QkGOvhHozZ6lwgFzQe3hTOxuFOv8wxg4E_vzruLMgSRmapHGuTi57qTYFWIJNb-VSUa_Nz__t6aXOaXYckO2nvzN2rp1qeTIEKrhLaC_nV5gZpOB2fnBC2Yr1KasERpdDO7I0xc4dqdLQXQRxrWgP6lKlkGKHziCrkvLEWqC7mXrRsS23m-qv4pELm0MikIqf-fl4wmwj7g42769APwAuoQdIgMnUOx2rT1ewkcW72py3wveX96oomdDyvIM6_y5uYALsTymc0xxr1yZOT9Gseypbjm-sa7byVaSbw2s9g\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Chromium\\\\\\\";v=\\\\\\\"104\\\\\\\", \\\\\\\" Not A;Brand\\\\\\\";v=\\\\\\\"99\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"104\\\\\\\"\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"Windows\\\\\\\"\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.9\\\",\\\"Content-Length\\\":\\\"2\\\",\\\"account\\\":\\\"1000000\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"X-Frame-Options\\\":\\\"deny\\\",\\\"Cache-Control\\\":\\\"no-cache, no-store, must-revalidate, pre-check=0, post-check=0\\\",\\\"Server\\\":\\\"AKTO server\\\",\\\"X-Content-Type-Options\\\":\\\"nosniff\\\",\\\"Content-Encoding\\\":\\\"gzip\\\",\\\"Vary\\\":\\\"Accept-Encoding, User-Agent\\\",\\\"Content-Length\\\":\\\"150\\\",\\\"X-XSS-Protection\\\":\\\"1\\\",\\\"Content-Language\\\":\\\"en-US\\\",\\\"Date\\\":\\\"Tue, 30 Aug 2022 17:22:40 GMT\\\",\\\"Content-Type\\\":\\\"application/json;charset=utf-8\\\"}\",\"time\":\"1661880160\",\"contentType\":\"application/json;charset=utf-8\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; diff --git a/apps/dashboard/src/main/java/com/akto/utils/Utils.java b/apps/dashboard/src/main/java/com/akto/utils/Utils.java index caa76cdf88..8a8c9403f2 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/Utils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/Utils.java @@ -475,13 +475,14 @@ public static void pushDataToKafka(int apiCollectionId, String topic, List initialNodes = new HashMap<>(); - public Map resultNodes = new HashMap<>(); +// public Map initialNodes = new HashMap<>(); + public Map resultNodes = new HashMap<>(); public void syncWithDb() { List nodes = new ArrayList<>(resultNodes.values()); @@ -30,35 +36,63 @@ public void syncWithDb() { } } - Queue queue = new LinkedList<>(); - Set done = new HashSet<>(); + Queue queue = new LinkedList<>(); + Set done = new HashSet<>(); - public void addToQueue(int nodeId) { - queue.add(nodeId); - done.add(nodeId); + public void addToQueue(ApiInfo.ApiInfoKey apiInfoKey) { + queue.add(apiInfoKey); + done.add(apiInfoKey); } + public List findL0Apis() { + int limit = 100; + ObjectId lastId = null; + Set neverL0 = new HashSet<>(); + Set maybeL0 = new HashSet<>(); + while (true) { + Bson filter = lastId == null ? new BasicDBObject() : Filters.gt(ID, lastId); + List dependencyNodeList = DependencyNodeDao.instance.findAll(filter, 0, limit, Sorts.ascending(ID)); + for (DependencyNode dependencyNode: dependencyNodeList) { + lastId = dependencyNode.getId(); + + ApiInfo.ApiInfoKey req = new ApiInfo.ApiInfoKey(Integer.parseInt(dependencyNode.getApiCollectionIdReq()), dependencyNode.getUrlReq(), URLMethods.Method.fromString(dependencyNode.getMethodReq())); + neverL0.add(req); + maybeL0.remove(req); + + ApiInfo.ApiInfoKey resp = new ApiInfo.ApiInfoKey(Integer.parseInt(dependencyNode.getApiCollectionIdResp()), dependencyNode.getUrlResp(), URLMethods.Method.fromString(dependencyNode.getMethodResp())); + if (!neverL0.contains(resp)) maybeL0.add(resp); + } + if (dependencyNodeList.size() < limit) break; + } + + return new ArrayList<>(maybeL0); + } + + static String ID = "_id"; public void run() { - initialNodes = new HashMap<>(); resultNodes = new HashMap<>(); queue = new LinkedList<>(); done = new HashSet<>(); - List dependencyNodeList = DependencyNodeDao.instance.findAll(new BasicDBObject()); - - for (DependencyNode dependencyNode: dependencyNodeList) { - fillNodes(dependencyNode); // to fill who is giving data to whom - fillResultNodes(dependencyNode); // to fill who is receiving data from whom + ObjectId lastId = null; + int limit = 100; + while (true) { + Bson filter = lastId == null ? new BasicDBObject() : Filters.gt(ID, lastId); + List dependencyNodeList = DependencyNodeDao.instance.findAll(filter, 0, limit, Sorts.ascending(ID)); + for (DependencyNode dependencyNode: dependencyNodeList) { + lastId = dependencyNode.getId(); + fillResultNodes(dependencyNode); // to fill who is receiving data from whom + } + if (dependencyNodeList.size() < limit) break; } + List l0Apis = findL0Apis(); + // build initial queue // depth 0 for initial queue - for (ReverseNode reverseNode: initialNodes.values()) { - int id = reverseNode.hashCode(); - if (resultNodes.containsKey(id)) continue; // to check if the nodes is getting data from anywhere. If yes then don't add to queue now. - addToQueue(id); - Node resultNodeForZeroLevel = new Node(reverseNode.getApiCollectionId(), reverseNode.getUrl(), reverseNode.getMethod(), new HashMap<>()); - resultNodes.put(id, resultNodeForZeroLevel); // we want to still store level 0 nodes in db + for (ApiInfo.ApiInfoKey apiInfoKey: l0Apis) { + addToQueue(apiInfoKey); + resultNodes.put(apiInfoKey, new Node(apiInfoKey.getApiCollectionId()+"", apiInfoKey.getUrl(), apiInfoKey.getMethod().name(), new HashMap<>())); // we want to still store level 0 nodes in db } parseTree(); // pops node from queue and fill all the child APIs of that particular @@ -66,25 +100,56 @@ public void run() { int maxCycles = 20; while (maxCycles > 0) { maxCycles -= 1; - Integer nodeToBeMarkedDone = markNodeAsDone(); + ApiInfo.ApiInfoKey nodeToBeMarkedDone = markNodeAsDone(); if (nodeToBeMarkedDone == null) return; addToQueue(nodeToBeMarkedDone); parseTree(); } } + public static ReverseNode buildReverseNode(ApiInfo.ApiInfoKey apiInfoKey) { + List dependencyNodes = DependencyNodeDao.instance.findAll( + Filters.and( + Filters.eq(DependencyNode.API_COLLECTION_ID_RESP, apiInfoKey.getApiCollectionId()+""), + Filters.eq(DependencyNode.URL_RESP, apiInfoKey.getUrl()), + Filters.eq(DependencyNode.METHOD_RESP, apiInfoKey.getMethod().name()) + ) + ); + + ReverseNode reverseNode = new ReverseNode( + apiInfoKey.getApiCollectionId()+"", apiInfoKey.getUrl(), apiInfoKey.getMethod().name(), new HashMap<>() + ); + for (DependencyNode dependencyNode: dependencyNodes) { + for (DependencyNode.ParamInfo paramInfo: dependencyNode.getParamInfos()) { + String paramResp = paramInfo.getResponseParam(); + ReverseConnection reverseConnection = reverseNode.getReverseConnections().getOrDefault(paramResp, new ReverseConnection(paramResp, new ArrayList<>())); + + String paramReq = paramInfo.getRequestParam(); + reverseConnection.getReverseEdges().add( + new ReverseEdge( + dependencyNode.getApiCollectionIdReq(), dependencyNode.getUrlReq(), dependencyNode.getMethodReq(), paramReq, paramInfo.getCount(), paramInfo.getIsUrlParam(), paramInfo.getIsHeader() + ) + ); + + reverseNode.getReverseConnections().put(paramResp, reverseConnection); + } + } + + return reverseNode; + } + public void parseTree() { // pop queue until empty while (!queue.isEmpty()) { - int nodeId = queue.remove(); - ReverseNode reverseNode = initialNodes.get(nodeId); // reverse node basically says this API is sending data to what other APIs + ApiInfo.ApiInfoKey apiInfoKey = queue.remove(); + ReverseNode reverseNode = buildReverseNode(apiInfoKey); - if (reverseNode == null || reverseNode.getReverseConnections() == null) { + if (reverseNode.getReverseConnections() == null) { // this means the node is not giving data to anyone so next node please! continue; } - Node nodeFromQueue = resultNodes.get(nodeId); + Node nodeFromQueue = resultNodes.get(apiInfoKey); nodeFromQueue.fillMaxDepth(); int depth = nodeFromQueue.getMaxDepth(); @@ -92,9 +157,9 @@ public void parseTree() { for (ReverseConnection reverseConnection: reverseConnections.values()) { for (ReverseEdge reverseEdge: reverseConnection.getReverseEdges()) { // resultNode is basically find which node is receiving the data and fill the edge with that particular parameter - Node resultNode = resultNodes.get(Objects.hash(reverseEdge.getApiCollectionId(), reverseEdge.getUrl(), reverseEdge.getMethod())); - int key = resultNode.hashCode(); - if (done.contains(key)) continue; + ApiInfo.ApiInfoKey resultApiInfoKey = new ApiInfo.ApiInfoKey(Integer.parseInt(reverseEdge.getApiCollectionId()), reverseEdge.getUrl(), URLMethods.Method.fromString(reverseEdge.getMethod())); + Node resultNode = resultNodes.get(resultApiInfoKey); + if (done.contains(resultApiInfoKey)) continue; Edge edge = new Edge(reverseNode.getApiCollectionId(), reverseNode.getUrl(), reverseNode.getMethod(), reverseConnection.getParam(), reverseEdge.getIsHeader(), reverseEdge.getCount(), depth + 1); String requestParam = reverseEdge.getParam(); @@ -113,14 +178,23 @@ public void parseTree() { boolean flag = isDone(resultNode); // a node is marked done when all parameters have found a parent if (flag) { - addToQueue(key); + addToQueue(resultApiInfoKey); } } } } } - public Integer markNodeAsDone() { + static String generateStringFromApiInfoKey(ApiInfo.ApiInfoKey apiInfoKey) { + return apiInfoKey.getApiCollectionId() + "$" + apiInfoKey.getUrl() + "$" + apiInfoKey.getMethod().name(); + } + + static ApiInfo.ApiInfoKey generateApiInfoKeyFromString(String apiInfoKeyString) { + String[] split = apiInfoKeyString.split("\\$"); + return new ApiInfo.ApiInfoKey(Integer.parseInt(split[0]), split[1], URLMethods.Method.fromString(split[2])); + } + + public ApiInfo.ApiInfoKey markNodeAsDone() { List nodes = new ArrayList<>(resultNodes.values()); // incompleteNodes are the nodes which have some parameters for which we were not able to figure out the parent @@ -137,15 +211,16 @@ public Integer markNodeAsDone() { // we add all the incomplete nodes to a graph Graph directedGraph = new DefaultDirectedGraph<>(DefaultEdge.class); for (Node node: incompleteNodes) { - directedGraph.addVertex(node.hashCode()+""); + ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), URLMethods.Method.fromString(node.getMethod())); + directedGraph.addVertex(generateStringFromApiInfoKey(apiInfoKey)); } // Building edges for (Node node: incompleteNodes) { - int nodeId = node.hashCode(); // reverse node basically says this API is sending data to what other APIs - ReverseNode reverseNode = initialNodes.get(nodeId); - if (reverseNode == null || reverseNode.getReverseConnections() == null) { + ApiInfo.ApiInfoKey parentApi = new ApiInfo.ApiInfoKey(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), URLMethods.Method.fromString(node.getMethod())); + ReverseNode reverseNode = buildReverseNode(parentApi); + if (reverseNode.getReverseConnections() == null) { // this means the node is not giving data to anyone so next node please! continue; } @@ -153,9 +228,9 @@ public Integer markNodeAsDone() { Map reverseConnections = reverseNode.getReverseConnections(); for (ReverseConnection reverseConnection: reverseConnections.values()) { for (ReverseEdge reverseEdge: reverseConnection.getReverseEdges()) { - int id = Objects.hash(reverseEdge.getApiCollectionId(), reverseEdge.getUrl(), reverseEdge.getMethod()); + ApiInfo.ApiInfoKey childApi = new ApiInfo.ApiInfoKey(Integer.parseInt(reverseEdge.getApiCollectionId()), reverseEdge.getUrl(), URLMethods.Method.fromString(reverseEdge.getMethod())); - Node resultNode = resultNodes.get(id); + Node resultNode = resultNodes.get(childApi); Map connections = resultNode.getConnections(); String param = reverseEdge.getParam(); @@ -170,8 +245,8 @@ public Integer markNodeAsDone() { List edges = connection.getEdges(); if (edges == null || !edges.isEmpty()) continue; // this is because if it already has an edge this means that parameter is not part of the loop, so skip it - if (directedGraph.containsVertex(id+"")) { // this check is added to make sure we are only making edges inside the loop - directedGraph.addEdge(nodeId+"", id+""); + if (directedGraph.containsVertex(generateStringFromApiInfoKey(childApi))) { // this check is added to make sure we are only making edges inside the loop + directedGraph.addEdge(generateStringFromApiInfoKey(parentApi), generateStringFromApiInfoKey(childApi)); } } } @@ -185,11 +260,12 @@ public Integer markNodeAsDone() { System.out.println("We found " + cycles.size() + " circular dependency"); int currentMinMissingCount = Integer.MAX_VALUE; - Integer result = null; + ApiInfo.ApiInfoKey result = null; for (List cycle : cycles) { - for (String nodeId: cycle) { - int key = Integer.parseInt(nodeId); - Node node = resultNodes.get(key); + for (String apiInfoKeyString: cycle) { + ApiInfo.ApiInfoKey apiInfoKey = generateApiInfoKeyFromString(apiInfoKeyString); + + Node node = resultNodes.get(apiInfoKey); Map connections = node.getConnections(); int missing = 0; int filled = 0; @@ -202,11 +278,11 @@ public Integer markNodeAsDone() { } } - if (missing == 0) return key; // this means except headers all values have been filled. + if (missing == 0) return apiInfoKey; // this means except headers all values have been filled. if (filled == 0) continue; - if (missing < currentMinMissingCount) result = key; + if (missing < currentMinMissingCount) result = apiInfoKey; } } @@ -226,33 +302,6 @@ public static boolean isDone(Node node) { return flag; } - public void fillNodes(DependencyNode dependencyNode) { - ReverseNode reverseNode = new ReverseNode( - dependencyNode.getApiCollectionIdResp(), - dependencyNode.getUrlResp(), - dependencyNode.getMethodResp(), - new HashMap<>() - ); - - Integer key = reverseNode.hashCode(); - reverseNode = initialNodes.getOrDefault(key, reverseNode); - - for (DependencyNode.ParamInfo paramInfo: dependencyNode.getParamInfos()) { - String paramResp = paramInfo.getResponseParam(); - ReverseConnection reverseConnection = reverseNode.getReverseConnections().getOrDefault(paramResp, new ReverseConnection(paramResp, new ArrayList<>())); - - String paramReq = paramInfo.getRequestParam(); - reverseConnection.getReverseEdges().add( - new ReverseEdge( - dependencyNode.getApiCollectionIdReq(), dependencyNode.getUrlReq(), dependencyNode.getMethodReq(), paramReq, paramInfo.getCount(), paramInfo.getIsUrlParam(), paramInfo.getIsHeader() - ) - ); - - reverseNode.getReverseConnections().put(paramResp, reverseConnection); - } - - initialNodes.put(key, reverseNode); - } public void fillResultNodes(DependencyNode dependencyNode) { Node node = new Node( @@ -262,7 +311,7 @@ public void fillResultNodes(DependencyNode dependencyNode) { new HashMap<>() ); - Integer key = node.hashCode(); + ApiInfo.ApiInfoKey key = new ApiInfo.ApiInfoKey(Integer.parseInt(dependencyNode.getApiCollectionIdReq()), dependencyNode.getUrlReq(), URLMethods.Method.fromString(dependencyNode.getMethodReq())); node = resultNodes.getOrDefault(key, node); for (DependencyNode.ParamInfo paramInfo: dependencyNode.getParamInfos()) { From a2a7fa33a4d7ad5fe40d9915ba6c6c14d5718282 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Fri, 8 Mar 2024 16:50:31 +0530 Subject: [PATCH 02/13] get all dependency nodes data once --- .../dto/dependency_flow/DependencyFlow.java | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/libs/dao/src/main/java/com/akto/dto/dependency_flow/DependencyFlow.java b/libs/dao/src/main/java/com/akto/dto/dependency_flow/DependencyFlow.java index 3db0e2603a..0277addcc0 100644 --- a/libs/dao/src/main/java/com/akto/dto/dependency_flow/DependencyFlow.java +++ b/libs/dao/src/main/java/com/akto/dto/dependency_flow/DependencyFlow.java @@ -21,7 +21,6 @@ public class DependencyFlow { -// public Map initialNodes = new HashMap<>(); public Map resultNodes = new HashMap<>(); public void syncWithDb() { @@ -44,28 +43,15 @@ public void addToQueue(ApiInfo.ApiInfoKey apiInfoKey) { done.add(apiInfoKey); } - public List findL0Apis() { - int limit = 100; - ObjectId lastId = null; - Set neverL0 = new HashSet<>(); - Set maybeL0 = new HashSet<>(); - while (true) { - Bson filter = lastId == null ? new BasicDBObject() : Filters.gt(ID, lastId); - List dependencyNodeList = DependencyNodeDao.instance.findAll(filter, 0, limit, Sorts.ascending(ID)); - for (DependencyNode dependencyNode: dependencyNodeList) { - lastId = dependencyNode.getId(); - - ApiInfo.ApiInfoKey req = new ApiInfo.ApiInfoKey(Integer.parseInt(dependencyNode.getApiCollectionIdReq()), dependencyNode.getUrlReq(), URLMethods.Method.fromString(dependencyNode.getMethodReq())); - neverL0.add(req); - maybeL0.remove(req); + public void findL0Apis(Set neverL0, Set maybeL0, List dependencyNodeList) { + for (DependencyNode dependencyNode: dependencyNodeList) { + ApiInfo.ApiInfoKey req = new ApiInfo.ApiInfoKey(Integer.parseInt(dependencyNode.getApiCollectionIdReq()), dependencyNode.getUrlReq(), URLMethods.Method.fromString(dependencyNode.getMethodReq())); + neverL0.add(req); + maybeL0.remove(req); - ApiInfo.ApiInfoKey resp = new ApiInfo.ApiInfoKey(Integer.parseInt(dependencyNode.getApiCollectionIdResp()), dependencyNode.getUrlResp(), URLMethods.Method.fromString(dependencyNode.getMethodResp())); - if (!neverL0.contains(resp)) maybeL0.add(resp); - } - if (dependencyNodeList.size() < limit) break; + ApiInfo.ApiInfoKey resp = new ApiInfo.ApiInfoKey(Integer.parseInt(dependencyNode.getApiCollectionIdResp()), dependencyNode.getUrlResp(), URLMethods.Method.fromString(dependencyNode.getMethodResp())); + if (!neverL0.contains(resp)) maybeL0.add(resp); } - - return new ArrayList<>(maybeL0); } static String ID = "_id"; @@ -76,9 +62,13 @@ public void run() { ObjectId lastId = null; int limit = 100; + Set neverL0 = new HashSet<>(); + Set maybeL0 = new HashSet<>(); + while (true) { Bson filter = lastId == null ? new BasicDBObject() : Filters.gt(ID, lastId); List dependencyNodeList = DependencyNodeDao.instance.findAll(filter, 0, limit, Sorts.ascending(ID)); + findL0Apis(neverL0, maybeL0, dependencyNodeList); for (DependencyNode dependencyNode: dependencyNodeList) { lastId = dependencyNode.getId(); fillResultNodes(dependencyNode); // to fill who is receiving data from whom @@ -86,7 +76,7 @@ public void run() { if (dependencyNodeList.size() < limit) break; } - List l0Apis = findL0Apis(); + List l0Apis = new ArrayList<>(maybeL0); // build initial queue // depth 0 for initial queue From b2234e2c0d24b934d5a753aba78554e70cb084e5 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Mon, 11 Mar 2024 15:59:16 +0530 Subject: [PATCH 03/13] store only the hashcode instead of complete value --- .../main/java/com/akto/dependency/store/HashSetStore.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/dependency/store/HashSetStore.java b/apps/api-runtime/src/main/java/com/akto/dependency/store/HashSetStore.java index b081b16016..cc11965df7 100644 --- a/apps/api-runtime/src/main/java/com/akto/dependency/store/HashSetStore.java +++ b/apps/api-runtime/src/main/java/com/akto/dependency/store/HashSetStore.java @@ -11,15 +11,15 @@ public HashSetStore(int maxCount) { this.maxCount = maxCount; } - private final Set set = new HashSet<>(); + private final Set set = new HashSet<>(); @Override public boolean contains(String val) { - return set.contains(val); + return set.contains(val.hashCode()); } @Override public boolean add(String val) { if (set.size() >= maxCount) return false; - return set.add(val); + return set.add(val.hashCode()); } } From 6fe16d547940c4156b8e159a959e2c769c55758d Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Tue, 9 Apr 2024 10:08:21 +0530 Subject: [PATCH 04/13] added filters --- .../components/tables/GithubServerTable.js | 40 ++++++++++++++---- .../dashboard/components/tables/transform.js | 41 ++++++++++++++++++- .../DependencyTable/DependencyTable.js | 3 ++ .../web/src/apps/main/PersistStore.js | 2 + .../web/polaris_web/web/src/util/func.js | 3 ++ 5 files changed, 80 insertions(+), 9 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js index c03148dea0..8cb4a87528 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js @@ -19,21 +19,36 @@ import { produce } from "immer" import values from "@/util/values" import transform from '../../pages/observe/transform'; import DropdownSearch from '../shared/DropdownSearch'; +import PersistStore from '../../../main/PersistStore'; +import tableFunc from './transform'; function GithubServerTable(props) { + const filtersMap = PersistStore(state => state.filtersMap) + const setFiltersMap = PersistStore(state => state.setFiltersMap) + const initialStateFilters = tableFunc.mergeFilters(props.appliedFilters || [], ((filtersMap[window.location.href] && filtersMap[window.location.href]['filters']) ? filtersMap[window.location.href]['filters'] : []),props.disambiguateLabel) + const { mode, setMode } = useSetIndexFiltersMode(props?.mode ? props.mode : IndexFiltersMode.Filtering); const [sortSelected, setSortSelected] = useState(props?.sortOptions?.length > 0 ? [props.sortOptions[0].value] : []); const [data, setData] = useState([]); const [total, setTotal] = useState([]); const [page, setPage] = useState(0); const pageLimit = props?.pageLimit || 20; - const [appliedFilters, setAppliedFilters] = useState(props.appliedFilters || []); + const [appliedFilters, setAppliedFilters] = useState(initialStateFilters); const [queryValue, setQueryValue] = useState(''); let filterOperators = props.headers.reduce((map, e) => { map[e.sortKey || e.value] = 'OR'; return map }, {}) const [currDateRange, dispatchCurrDateRange] = useReducer(produce((draft, action) => func.dateRangeReducer(draft, action)), values.ranges[3]); + const handleSelectedTab = (x) => { + const tableTabs = props.tableTabs ? props.tableTabs : props.tabs + if(tableTabs){ + const primitivePath = window.location.origin + window.location.pathname + const newUrl = primitivePath + "#" + tableTabs[x].content + window.history.replaceState(null, null, newUrl) + } + } + useEffect(() => { let [sortKey, sortOrder] = sortSelected.length == 0 ? ["", ""] : sortSelected[0].split(" "); let filters = props.headers.reduce((map, e) => { map[e.filterKey || e.value] = []; return map }, {}) @@ -45,6 +60,7 @@ function GithubServerTable(props) { tempData ? setData([...tempData.value]) : setData([]) tempData ? setTotal(tempData.total) : setTotal(0) } + handleSelectedTab(props?.selected) fetchData(); }, [sortSelected, appliedFilters, queryValue, page]) @@ -54,21 +70,24 @@ function GithubServerTable(props) { return filter.key != key }) props?.appliedFilters?.forEach((defaultAppliedFilter) => { - if (key == defaultAppliedFilter.key) { + if (key === defaultAppliedFilter.key) { temp.push(defaultAppliedFilter) } }) - if (key == "dateRange") { + if (key === "dateRange") { getDate({ type: "update", period: values.ranges[3] }); } else { setAppliedFilters(temp); + let tempFilters = filtersMap + tempFilters[window.location.href]['filters'] = temp + setFiltersMap(tempFilters) } } const changeAppliedFilters = (key, value) => { let temp = appliedFilters temp = temp.filter((filter) => { - return filter.key != key + return filter.key !== key }) if (value.length > 0 || Object.keys(value).length > 0) { temp.push({ @@ -79,6 +98,9 @@ function GithubServerTable(props) { }) } setPage(0); + let tempFilters = filtersMap + tempFilters[window.location.href]['filters'] = temp + setFiltersMap(tempFilters) setAppliedFilters(temp); }; @@ -164,8 +186,11 @@ function GithubServerTable(props) { } const handleFiltersClearAll = useCallback(() => { + setFiltersMap([]) + if(appliedFilters.length > 0 && appliedFilters.filter((x) => x.key === 'dateRange')){ + getDate({ type: "update", period: values.ranges[3] }); + } setAppliedFilters(props.appliedFilters || []) - getDate({ type: "update", period: values.ranges[3] }); }, []); const resourceIDResolver = (data) => { @@ -222,11 +247,10 @@ function GithubServerTable(props) { let tableHeightClass = props.increasedHeight ? "control-row" : (props.condensedHeight ? "condensed-row" : '') let tableClass = props.useNewRow ? "new-table" : (props.selectable ? "removeHeaderColor" : "hideTableHead") - return (
- {props.tabs && } + {props.tabs && {props?.onSelect(x); handleSelectedTab(x)}}>} {props.tabs && props.tabs[props.selected].component ? props.tabs[props.selected].component :
@@ -252,7 +276,7 @@ function GithubServerTable(props) { setMode={setMode} loading={props.loading || false} selected={props?.selected} - onSelect={props?.onSelect} + onSelect={(x) => {props?.onSelect(x); handleSelectedTab(x)}} />
{ + if (acc[key]) { + if(key === 'dateRange'){ + acc[key].value = this.mergeTimeRanges(acc[key].value, value) + }else{ + acc[key].value = [...new Set([...acc[key].value, ...value])]; + } + } else { + if(key === 'dateRange'){ + acc[key] = {key, value} + }else{ + acc[key] = { key, value: [...value] }; + } + } + return acc; + }, {}); + + return Object.keys(mergedByKey).map((key) => { + const obj = mergedByKey[key] + return { + ...obj, + label: labelFunc(obj.key, obj.value) + } + }) + }, + mergeTimeRanges(obj1, obj2) { + const sinceEpoch1 = Date.parse(obj1.since); + const untilEpoch1 = Date.parse(obj1.until); + const sinceEpoch2 = Date.parse(obj2.since); + const untilEpoch2 = Date.parse(obj2.until); + const minSinceEpoch = Math.min(sinceEpoch1, sinceEpoch2); + const maxUntilEpoch = Math.max(untilEpoch1, untilEpoch2); + const since = new Date(minSinceEpoch).toISOString(); + const until = new Date(maxUntilEpoch).toISOString(); + + return { since, until }; + } } export default tableFunc; \ No newline at end of file diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/DependencyTable.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/DependencyTable.js index 991be441b5..53132ae0f5 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/DependencyTable.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/DependencyTable.js @@ -9,6 +9,7 @@ import TableExpand from "./TableExpand"; import func from "../../../../../util/func"; import EditModal from "./EditModal"; import GlobalVarModal from "./GlobalVarModal"; +import { useLocation } from "react-router-dom"; const headers = [ { @@ -65,6 +66,7 @@ function DependencyTable() { const [editData, setEditData] = useState([]) const [globalVarActive, setGlobalVarActive] = useState(false) + const location = useLocation() const queryParams = new URLSearchParams(location.search); const apiCollectionIdsString = queryParams.get('col_ids') @@ -231,6 +233,7 @@ function DependencyTable() { headings={headers} useNewRow={true} condensedHeight={true} + tableId="dependency-table" /> ) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/main/PersistStore.js b/apps/dashboard/web/polaris_web/web/src/apps/main/PersistStore.js index 27a2b1fee7..f7bf0598d9 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/main/PersistStore.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/main/PersistStore.js @@ -22,6 +22,7 @@ const initialState = { lastFetchedSensitiveResp: [], // selectedSampleApi: {}, coverageMap:{}, + filtersMap:{}, }; let persistStore = (set) => ({ @@ -42,6 +43,7 @@ let persistStore = (set) => ({ setLastFetchedSensitiveResp: (lastFetchedSensitiveResp) => set({ lastFetchedSensitiveResp }), // setSelectedSampleApi: (selectedSampleApi) => set({selectedSampleApi: selectedSampleApi}), setCoverageMap:(coverageMap)=>{set({coverageMap: coverageMap})}, + setFiltersMap: (filtersMap) => set({ filtersMap }), resetAll: () => set(initialState), // Reset function }) diff --git a/apps/dashboard/web/polaris_web/web/src/util/func.js b/apps/dashboard/web/polaris_web/web/src/util/func.js index 7ea25451fb..377a816fcf 100644 --- a/apps/dashboard/web/polaris_web/web/src/util/func.js +++ b/apps/dashboard/web/polaris_web/web/src/util/func.js @@ -1071,6 +1071,9 @@ getDeprecatedEndpoints(apiInfoList, unusedEndpoints, apiCollectionId) { }, convertToDisambiguateLabelObj(value, convertObj, maxAllowed){ + if(!value || value.length === 0 || !Array.isArray(value)){ + return "" + } if (value.length > maxAllowed) { return `${value.slice(0, maxAllowed) .map(val => convertObj ? convertObj[val] : val) From d7b851546912d956586005ec459eaae13e9bd61b Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 11 Apr 2024 10:24:13 +0530 Subject: [PATCH 05/13] implementation of persisting sort done --- .../components/tables/GithubServerTable.js | 59 ++++++++++++++++--- .../dashboard/components/tables/transform.js | 33 +++++++++++ .../pages/issues/IssuesPage/IssuesPage.jsx | 9 +-- .../AllSensitiveData/AllSensitiveData.jsx | 12 ++-- .../SensitiveDataExposure.jsx | 5 +- .../api_collections/ApiCollections.jsx | 27 ++++++--- .../observe/api_collections/ApiEndpoints.jsx | 38 +++++++----- .../observe/api_collections/WorkflowTests.jsx | 5 +- .../api_collections/data/apiChanges.js | 30 ++++++---- .../integrations/webhooks/Webhooks.jsx | 5 +- .../SingleTestRunPage/SingleTestRunPage.js | 10 ++-- .../testing/TestRunsPage/TestRunsPage.js | 5 +- 12 files changed, 175 insertions(+), 63 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js index 8cb4a87528..eb019eb931 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js @@ -26,16 +26,20 @@ function GithubServerTable(props) { const filtersMap = PersistStore(state => state.filtersMap) const setFiltersMap = PersistStore(state => state.setFiltersMap) - const initialStateFilters = tableFunc.mergeFilters(props.appliedFilters || [], ((filtersMap[window.location.href] && filtersMap[window.location.href]['filters']) ? filtersMap[window.location.href]['filters'] : []),props.disambiguateLabel) - + const pageFiltersMap = filtersMap[window.location.href] + const initialStateFilters = tableFunc.mergeFilters(props.appliedFilters || [], (pageFiltersMap?.filters || []),props.disambiguateLabel) const { mode, setMode } = useSetIndexFiltersMode(props?.mode ? props.mode : IndexFiltersMode.Filtering); - const [sortSelected, setSortSelected] = useState(props?.sortOptions?.length > 0 ? [props.sortOptions[0].value] : []); + const [sortSelected, setSortSelected] = useState(tableFunc.getInitialSortSelected(props.sortOptions, pageFiltersMap)) const [data, setData] = useState([]); const [total, setTotal] = useState([]); const [page, setPage] = useState(0); const pageLimit = props?.pageLimit || 20; const [appliedFilters, setAppliedFilters] = useState(initialStateFilters); const [queryValue, setQueryValue] = useState(''); + + const [sortableColumns, setSortableColumns] = useState([]) + const [activeColumnSort, setActiveColumnSort] = useState({columnIndex: -1, sortDirection: 'descending'}) + let filterOperators = props.headers.reduce((map, e) => { map[e.sortKey || e.value] = 'OR'; return map }, {}) const [currDateRange, dispatchCurrDateRange] = useReducer(produce((draft, action) => func.dateRangeReducer(draft, action)), values.ranges[3]); @@ -49,8 +53,14 @@ function GithubServerTable(props) { } } + useEffect(()=> { + setAppliedFilters(initialStateFilters) + setSortSelected(tableFunc.getInitialSortSelected(props.sortOptions, pageFiltersMap)) + },[window.location.href]) + useEffect(() => { let [sortKey, sortOrder] = sortSelected.length == 0 ? ["", ""] : sortSelected[0].split(" "); + setActiveColumnSort(tableFunc.getColumnSort(sortSelected, props?.sortOptions)) let filters = props.headers.reduce((map, e) => { map[e.filterKey || e.value] = []; return map }, {}) appliedFilters.forEach((filter) => { filters[filter.key] = filter.value @@ -62,7 +72,28 @@ function GithubServerTable(props) { } handleSelectedTab(props?.selected) fetchData(); - }, [sortSelected, appliedFilters, queryValue, page]) + }, [sortSelected, appliedFilters, queryValue, page, pageFiltersMap]) + + useEffect(()=> { + setSortableColumns(tableFunc.getSortableChoices(props?.headers)) + },[props?.headers]) + + const handleSort = (col, dir) => { + let tempSortSelected = props?.sortOptions.filter(x => x.columnIndex === (col + 1)) + let sortVal = [tempSortSelected[0].value] + if(dir.includes("desc")){ + setSortSelected([tempSortSelected[1].value]) + sortVal = [tempSortSelected[1].value] + }else{ + setSortSelected([tempSortSelected[0].value]) + } + let copyFilters = filtersMap + copyFilters[window.location.href] = { + 'filters': pageFiltersMap?.filters || [], + 'sort': sortVal + } + setFiltersMap(copyFilters) + } const handleRemoveAppliedFilter = (key) => { let temp = appliedFilters @@ -79,7 +110,10 @@ function GithubServerTable(props) { } else { setAppliedFilters(temp); let tempFilters = filtersMap - tempFilters[window.location.href]['filters'] = temp + tempFilters[window.location.href] = { + 'filters': temp, + 'sort': pageFiltersMap?.sort || [] + } setFiltersMap(tempFilters) } } @@ -99,7 +133,10 @@ function GithubServerTable(props) { } setPage(0); let tempFilters = filtersMap - tempFilters[window.location.href]['filters'] = temp + tempFilters[window.location.href] = { + 'filters': temp, + 'sort': pageFiltersMap?.sort || [] + } setFiltersMap(tempFilters) setAppliedFilters(temp); }; @@ -186,11 +223,13 @@ function GithubServerTable(props) { } const handleFiltersClearAll = useCallback(() => { - setFiltersMap([]) + let tempFilters = pageFiltersMap + delete tempFilters.filters + setFiltersMap(tempFilters) if(appliedFilters.length > 0 && appliedFilters.filter((x) => x.key === 'dateRange')){ getDate({ type: "update", period: values.ranges[3] }); } - setAppliedFilters(props.appliedFilters || []) + setAppliedFilters([]) }, []); const resourceIDResolver = (data) => { @@ -298,6 +337,10 @@ function GithubServerTable(props) { bulkActions={props.selectable ? props.bulkActions && props.bulkActions(selectedResources) : []} promotedBulkActions={props.selectable ? props.promotedBulkActions && props.promotedBulkActions(selectedResources) : []} hasZebraStriping={props.hasZebraStriping || false} + sortable={sortableColumns} + sortColumnIndex={activeColumnSort.columnIndex} + sortDirection={activeColumnSort.sortDirection} + onSort={handleSort} > {rowMarkup} diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/transform.js index 7952a721d3..94eecca357 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/transform.js @@ -100,6 +100,39 @@ const tableFunc = { const until = new Date(maxUntilEpoch).toISOString(); return { since, until }; + }, + getSortableChoices(sortOptions){ + if(!sortOptions || sortOptions === undefined || sortOptions.length === 0){ + return [] + } + + let sortableColumns = [] + sortOptions.forEach((opt) => { + sortableColumns.push(opt.sortActive || false) + }) + return sortableColumns + }, + getColumnSort(sortSelected, sortOptions){ + if(!sortSelected || sortSelected.length === 0 || !sortOptions || sortOptions === undefined || sortOptions.length === 0){ + return {columnIndex: -1, sortDirection: 'descending'} + } + + const sortColumn = sortOptions.filter((x) => x.value === sortSelected[0])[0] + const sortDirection = sortSelected[0].split(" ")[1] === "asc" ? "ascending" : "descending" + + return { + columnIndex: sortColumn.columnIndex - 1, + sortDirection: sortDirection + } + }, + getInitialSortSelected(sortOptions, filtersMap){ + if(!sortOptions || sortOptions === undefined || sortOptions.length === 0){ + return {columnIndex: -1, sortDirection: 'descending'} + } + if(!filtersMap || filtersMap?.sort === undefined || filtersMap.sort.length === 0){ + return [sortOptions[0].value] + } + return filtersMap.sort } } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/issues/IssuesPage/IssuesPage.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/issues/IssuesPage/IssuesPage.jsx index b1d82d4f99..36b8f2f4bf 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/issues/IssuesPage/IssuesPage.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/issues/IssuesPage/IssuesPage.jsx @@ -38,6 +38,7 @@ const headers = [ { text: "Timestamp", value: "timestamp", + sortActive: true }, { text: "Endpoint", @@ -65,10 +66,10 @@ const headers = [ ] const sortOptions = [ - { label: 'Discovered time', value: 'timestamp asc', directionLabel: 'Newest', sortKey: 'timestamp' }, - { label: 'Discovered time', value: 'timestamp desc', directionLabel: 'Oldest', sortKey: 'timestamp' }, - { label: 'Issue', value: 'categoryName asc', directionLabel: 'A-Z', sortKey: 'categoryName' }, - { label: 'Issue', value: 'categoryName desc', directionLabel: 'Z-A', sortKey: 'categoryName' }, + { label: 'Discovered time', value: 'timestamp asc', directionLabel: 'Newest', sortKey: 'timestamp', columnIndex: 5 }, + { label: 'Discovered time', value: 'timestamp desc', directionLabel: 'Oldest', sortKey: 'timestamp', columnIndex: 5 }, + { label: 'Issue', value: 'categoryName asc', directionLabel: 'A-Z', sortKey: 'categoryName', columnIndex: 1 }, + { label: 'Issue', value: 'categoryName desc', directionLabel: 'Z-A', sortKey: 'categoryName', columnIndex: 1 }, ]; let filtersOptions = [ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx index a816668c03..ea4335ab11 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx @@ -24,6 +24,7 @@ const headers = [ value: "subType", showFilter:true, itemOrder: 1, + sortActive: true }, { text: "Custom type", @@ -43,15 +44,16 @@ const headers = [ }, { text:"Sensitive count", - value: "sensitiveCount" + value: "sensitiveCount", + sortActive: true } ] const sortOptions = [ - { label: 'Sensitive data', value: 'sensitiveCount asc', directionLabel: 'More exposure', sortKey: 'sensitiveCount' }, - { label: 'Sensitive data', value: 'sensitiveCount desc', directionLabel: 'Less exposure', sortKey: 'sensitiveCount' }, - { label: 'Data type', value: 'subType asc', directionLabel: 'A-Z', sortKey: 'subType' }, - { label: 'Data type', value: 'subType desc', directionLabel: 'Z-A', sortKey: 'subType' }, + { label: 'Sensitive data', value: 'sensitiveCount asc', directionLabel: 'More exposure', sortKey: 'sensitiveCount', columnIndex: 6 }, + { label: 'Sensitive data', value: 'sensitiveCount desc', directionLabel: 'Less exposure', sortKey: 'sensitiveCount', columnIndex: 6 }, + { label: 'Data type', value: 'subType asc', directionLabel: 'A-Z', sortKey: 'subType', columnIndex: 2 }, + { label: 'Data type', value: 'subType desc', directionLabel: 'Z-A', sortKey: 'subType', columnIndex: 2 }, ]; const resourceName = { diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx index 64d73e4701..c0e60c7af5 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx @@ -38,6 +38,7 @@ const headers = [ { text: "Timestamp", value: "timestamp", + sortActive: true }, { text: "Location", @@ -55,8 +56,8 @@ const headers = [ ] const sortOptions = [ - { label: 'Discovered time', value: 'timestamp asc', directionLabel: 'Newest', sortKey: 'timestamp' }, - { label: 'Discovered time', value: 'timestamp desc', directionLabel: 'Oldest', sortKey: 'timestamp' }, + { label: 'Discovered time', value: 'timestamp asc', directionLabel: 'Newest', sortKey: 'timestamp', columnIndex:4 }, + { label: 'Discovered time', value: 'timestamp desc', directionLabel: 'Oldest', sortKey: 'timestamp', columnIndex:4 }, ]; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx index e343fb9175..5640847d14 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx @@ -28,10 +28,12 @@ const headers = [ text: "Total endpoints", value: "endpoints", isText: CellType.TEXT, + sortActive: true }, { title: 'Risk score', value: 'riskScoreComp', + sortActive: true }, { title: 'Test coverage', @@ -61,16 +63,26 @@ const headers = [ text: 'Last traffic seen', value: 'lastTraffic', isText: CellType.TEXT, + sortActive: true + }, + { + title: 'Discovered', + text: 'Discovered', + value: 'discovered', + isText: CellType.TEXT, + sortActive: true } ] const sortOptions = [ - { label: 'Risk Score', value: 'score asc', directionLabel: 'High risk', sortKey: 'riskScore' }, - { label: 'Risk Score', value: 'score desc', directionLabel: 'Low risk', sortKey: 'riskScore' }, - { label: 'Discovered', value: 'detected asc', directionLabel: 'Recent first', sortKey: 'startTs' }, - { label: 'Discovered', value: 'detected desc', directionLabel: 'Oldest first', sortKey: 'startTs' }, - { label: 'Endpoints', value: 'endpoints asc', directionLabel: 'More', sortKey: 'endpoints' }, - { label: 'Endpoints', value: 'endpoints desc', directionLabel: 'Less', sortKey: 'endpoints' }, + { label: 'Risk Score', value: 'score asc', directionLabel: 'High risk', sortKey: 'riskScore', columnIndex: 3 }, + { label: 'Risk Score', value: 'score desc', directionLabel: 'Low risk', sortKey: 'riskScore' , columnIndex: 3}, + { label: 'Discovered', value: 'discovered asc', directionLabel: 'Recent first', sortKey: 'startTs', columnIndex: 9 }, + { label: 'Discovered', value: 'discovered desc', directionLabel: 'Oldest first', sortKey: 'startTs' , columnIndex: 9}, + { label: 'Endpoints', value: 'endpoints asc', directionLabel: 'More', sortKey: 'endpoints', columnIndex: 2 }, + { label: 'Endpoints', value: 'endpoints desc', directionLabel: 'Less', sortKey: 'endpoints' , columnIndex: 2}, + { label: 'Last traffic seen', value: 'detected asc', directionLabel: 'Recent first', sortKey: 'detected', columnIndex: 8 }, + { label: 'Last traffic seen', value: 'detected desc', directionLabel: 'Oldest first', sortKey: 'detected' , columnIndex: 8}, ]; @@ -99,7 +111,8 @@ const convertToNewData = (collectionsArr, sensitiveInfoMap, severityInfoMap, cov sensitiveInRespTypes: sensitiveInfoMap[c.id] ? sensitiveInfoMap[c.id] : [], severityInfo: severityInfoMap[c.id] ? severityInfoMap[c.id] : {}, detected: func.prettifyEpoch(trafficInfoMap[c.id] || 0), - riskScore: riskScoreMap[c.id] ? riskScoreMap[c.id] : 0 + riskScore: riskScoreMap[c.id] ? riskScoreMap[c.id] : 0, + discovered: func.prettifyEpoch(c.startTs || 0), } }) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index 9fa0ae3923..f21c48caa4 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -31,13 +31,15 @@ const headings = [ text: "Endpoint", value: "endpointComp", title: "Api endpoints", - textValue: "endpoint" + textValue: "endpoint", + sortActive: true }, { text: "Risk score", title: "Risk score", value: "riskScoreComp", - textValue: "riskScore" + textValue: "riskScore", + sortActive: true }, { text: "Hostname", @@ -52,13 +54,15 @@ const headings = [ title: 'Access type', showFilter: true, type: CellType.TEXT, + sortActive: true }, { text: 'Auth Type', title: 'Auth type', value: 'auth_type', showFilter: true, - textValue: 'authTypeTag' + textValue: 'authTypeTag', + sortActive: true }, { text: 'Sensitive Params', @@ -73,7 +77,8 @@ const headings = [ title: 'Last seen', value: 'last_seen', isText: true, - type: CellType.TEXT + type: CellType.TEXT, + sortActive: true } ] @@ -83,22 +88,23 @@ headers.push({ filterKey: 'method', showFilter: true, textValue: 'method', + sortActive: true }) const sortOptions = [ - { label: 'Risk Score', value: 'riskScore asc', directionLabel: 'Highest', sortKey: 'riskScore'}, - { label: 'Risk Score', value: 'riskScore desc', directionLabel: 'Lowest', sortKey: 'riskScore'}, - { label: 'Method', value: 'method asc', directionLabel: 'A-Z', sortKey: 'method' }, - { label: 'Method', value: 'method desc', directionLabel: 'Z-A', sortKey: 'method' }, - { label: 'Endpoint', value: 'endpoint asc', directionLabel: 'A-Z', sortKey: 'endpoint' }, - { label: 'Endpoint', value: 'endpoint desc', directionLabel: 'Z-A', sortKey: 'endpoint' }, - { label: 'Auth Type', value: 'auth_type asc', directionLabel: 'A-Z', sortKey: 'auth_type' }, - { label: 'Auth Type', value: 'auth_type desc', directionLabel: 'Z-A', sortKey: 'auth_type' }, - { label: 'Access Type', value: 'access_type asc', directionLabel: 'A-Z', sortKey: 'access_type' }, - { label: 'Access Type', value: 'access_type desc', directionLabel: 'Z-A', sortKey: 'access_type' }, - { label: 'Last seen', value: 'lastSeenTs asc', directionLabel: 'Newest', sortKey: 'lastSeenTs' }, - { label: 'Last seen', value: 'lastSeenTs desc', directionLabel: 'Oldest', sortKey: 'lastSeenTs' }, + { label: 'Risk Score', value: 'riskScore asc', directionLabel: 'Highest', sortKey: 'riskScore', columnIndex: 2}, + { label: 'Risk Score', value: 'riskScore desc', directionLabel: 'Lowest', sortKey: 'riskScore', columnIndex: 2}, + { label: 'Method', value: 'method asc', directionLabel: 'A-Z', sortKey: 'method', columnIndex: 8 }, + { label: 'Method', value: 'method desc', directionLabel: 'Z-A', sortKey: 'method', columnIndex: 8 }, + { label: 'Endpoint', value: 'endpoint asc', directionLabel: 'A-Z', sortKey: 'endpoint', columnIndex: 1 }, + { label: 'Endpoint', value: 'endpoint desc', directionLabel: 'Z-A', sortKey: 'endpoint', columnIndex: 1 }, + { label: 'Auth Type', value: 'auth_type asc', directionLabel: 'A-Z', sortKey: 'auth_type', columnIndex: 5 }, + { label: 'Auth Type', value: 'auth_type desc', directionLabel: 'Z-A', sortKey: 'auth_type', columnIndex: 5 }, + { label: 'Access Type', value: 'access_type asc', directionLabel: 'A-Z', sortKey: 'access_type', columnIndex: 4 }, + { label: 'Access Type', value: 'access_type desc', directionLabel: 'Z-A', sortKey: 'access_type', columnIndex: 4 }, + { label: 'Last seen', value: 'lastSeenTs asc', directionLabel: 'Newest', sortKey: 'lastSeenTs', columnIndex: 7 }, + { label: 'Last seen', value: 'lastSeenTs desc', directionLabel: 'Oldest', sortKey: 'lastSeenTs', columnIndex: 7 }, ]; function ApiEndpoints() { diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/WorkflowTests.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/WorkflowTests.jsx index 1c2f8cfd9b..95aa0191d2 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/WorkflowTests.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/WorkflowTests.jsx @@ -49,6 +49,7 @@ function WorkflowTests({ apiCollectionId, endpointsList }) { value: "author", showFilter: true, itemCell: 2, + sortActive: true, }, { text: "Created", @@ -65,8 +66,8 @@ function WorkflowTests({ apiCollectionId, endpointsList }) { ] const sortOptions = [ - { label: 'Author', value: 'author asc', directionLabel: 'A-Z', sortKey: 'author' }, - { label: 'Author', value: 'author desc', directionLabel: 'Z-A', sortKey: 'author' }, + { label: 'Author', value: 'author asc', directionLabel: 'A-Z', sortKey: 'author', columnIndex: 2 }, + { label: 'Author', value: 'author desc', directionLabel: 'Z-A', sortKey: 'author', columnIndex: 2 }, ]; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/data/apiChanges.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/data/apiChanges.js index 362faa0306..04e6eb6f49 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/data/apiChanges.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/data/apiChanges.js @@ -5,6 +5,7 @@ const endpointHeadings = [ text: "Endpoint", value: "endpointComp", title: "Api endpoints", + sortActive: true }, { text: "Risk score", @@ -31,6 +32,7 @@ const endpointHeadings = [ title: 'Access type', showFilter: true, type: CellType.TEXT, + sortActive: true }, { text: 'Auth Type', @@ -38,6 +40,7 @@ const endpointHeadings = [ value: 'auth_type', showFilter: true, type: CellType.TEXT, + sortActive: true }, { text: 'Sensitive Params', @@ -50,6 +53,7 @@ const endpointHeadings = [ title: 'Last seen', value: 'last_seen', type: CellType.TEXT, + sortActive: true }, ] @@ -73,7 +77,8 @@ const newParametersHeaders = [ sortKey: 'timestamp', showFilterMenu: true, type: CellType.TEXT, - maxWidth: '120px' + maxWidth: '120px', + sortActive: true }, { text: "Endpoint", @@ -127,22 +132,25 @@ const methodObj = [{ filterKey: 'method', showFilter: true, textValue: 'method', + sortActive: true }] const endpointSortOptions = [ - { label: 'Method', value: 'method asc', directionLabel: 'A-Z', sortKey: 'method' }, - { label: 'Method', value: 'method desc', directionLabel: 'Z-A', sortKey: 'method' }, - { label: 'Endpoint', value: 'endpoint asc', directionLabel: 'A-Z', sortKey: 'url' }, - { label: 'Endpoint', value: 'endpoint desc', directionLabel: 'Z-A', sortKey: 'url' }, - { label: 'Auth Type', value: 'auth_type asc', directionLabel: 'A-Z', sortKey: 'auth_type' }, - { label: 'Auth Type', value: 'auth_type desc', directionLabel: 'Z-A', sortKey: 'auth_type' }, - { label: 'Access Type', value: 'access_type asc', directionLabel: 'A-Z', sortKey: 'access_type' }, - { label: 'Access Type', value: 'access_type desc', directionLabel: 'Z-A', sortKey: 'access_type' }, + { label: 'Last seen', value: 'lastSeenTs asc', directionLabel: 'Recent first', sortKey: 'lastSeenTs', columnIndex: 8 }, + { label: 'Last seen', value: 'lastSeenTs desc', directionLabel: 'Oldest first', sortKey: 'lastSeenTs', columnIndex: 8 }, + { label: 'Method', value: 'method asc', directionLabel: 'A-Z', sortKey: 'method', columnIndex: 9 }, + { label: 'Method', value: 'method desc', directionLabel: 'Z-A', sortKey: 'method', columnIndex: 9 }, + { label: 'Endpoint', value: 'endpoint asc', directionLabel: 'A-Z', sortKey: 'url', columnIndex: 1 }, + { label: 'Endpoint', value: 'endpoint desc', directionLabel: 'Z-A', sortKey: 'url', columnIndex: 1 }, + { label: 'Auth Type', value: 'auth_type asc', directionLabel: 'A-Z', sortKey: 'auth_type', columnIndex: 6 }, + { label: 'Auth Type', value: 'auth_type desc', directionLabel: 'Z-A', sortKey: 'auth_type', columnIndex: 6 }, + { label: 'Access Type', value: 'access_type asc', directionLabel: 'A-Z', sortKey: 'access_type', columnIndex: 5 }, + { label: 'Access Type', value: 'access_type desc', directionLabel: 'Z-A', sortKey: 'access_type', columnIndex: 5 }, ]; const parameterSortOptions = [ - { label: 'Discovered time', value: 'timestamp asc', directionLabel: 'Newest', sortKey: 'timestamp' }, - { label: 'Discovered time', value: 'timestamp desc', directionLabel: 'Oldest', sortKey: 'timestamp' }, + { label: 'Discovered time', value: 'timestamp asc', directionLabel: 'Newest', sortKey: 'timestamp', columnIndex: 3}, + { label: 'Discovered time', value: 'timestamp desc', directionLabel: 'Oldest', sortKey: 'timestamp', columnIndex: 3 }, ]; let paramFilters = [ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/webhooks/Webhooks.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/webhooks/Webhooks.jsx index 2f71bfe60c..f9aed57fae 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/webhooks/Webhooks.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/webhooks/Webhooks.jsx @@ -58,6 +58,7 @@ function Webhooks() { value: "webhookName", showFilter: true, itemOrder: 1, + sortActive: true }, { text: "Create time", @@ -80,8 +81,8 @@ function Webhooks() { ] const sortOptions = [ - { label: 'Name', value: 'webhookName asc', directionLabel: 'A-Z', sortKey: 'webhookName' }, - { label: 'Name', value: 'webhookName desc', directionLabel: 'Z-A', sortKey: 'webhookName' }, + { label: 'Name', value: 'webhookName asc', directionLabel: 'A-Z', sortKey: 'webhookName', columnIndex: 1 }, + { label: 'Name', value: 'webhookName desc', directionLabel: 'Z-A', sortKey: 'webhookName', columnIndex: 1 }, ]; async function handleWebhookStatusChange(id, status) { diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js index 726d2280db..3ac52f96fd 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js @@ -37,6 +37,7 @@ let headers = [ { title: 'Severity', value: 'severityComp', + sortActive: true }, { value: 'testCategory', @@ -55,6 +56,7 @@ let headers = [ { value: "scanned_time_comp", title: 'Scanned', + sortActive: true }, { title: '', @@ -63,10 +65,10 @@ let headers = [ ] const sortOptions = [ - { label: 'Severity', value: 'severity asc', directionLabel: 'Highest severity', sortKey: 'total_severity' }, - { label: 'Severity', value: 'severity desc', directionLabel: 'Lowest severity', sortKey: 'total_severity' }, - { label: 'Run time', value: 'time asc', directionLabel: 'Newest run', sortKey: 'endTimestamp' }, - { label: 'Run time', value: 'time desc', directionLabel: 'Oldest run', sortKey: 'endTimestamp' }, + { label: 'Severity', value: 'severity asc', directionLabel: 'Highest severity', sortKey: 'total_severity', columnIndex: 2}, + { label: 'Severity', value: 'severity desc', directionLabel: 'Lowest severity', sortKey: 'total_severity', columnIndex: 2 }, + { label: 'Run time', value: 'time asc', directionLabel: 'Newest run', sortKey: 'endTimestamp', columnIndex: 5 }, + { label: 'Run time', value: 'time desc', directionLabel: 'Oldest run', sortKey: 'endTimestamp', columnIndex: 5 }, ]; const resourceName = { diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestRunsPage.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestRunsPage.js index f94ab801aa..01b5af7574 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestRunsPage.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestRunsPage.js @@ -65,6 +65,7 @@ let headers = [ title: 'Status', itemOrder: 3, type: CellType.TEXT, + sortActive: true }, { title: '', @@ -73,8 +74,8 @@ let headers = [ ] const sortOptions = [ - { label: 'Run time', value: 'endTimestamp asc', directionLabel: 'Newest run', sortKey: 'endTimestamp' }, - { label: 'Run time', value: 'endTimestamp desc', directionLabel: 'Oldest run', sortKey: 'endTimestamp' } + { label: 'Run time', value: 'endTimestamp asc', directionLabel: 'Newest run', sortKey: 'endTimestamp', columnIndex: 4 }, + { label: 'Run time', value: 'endTimestamp desc', directionLabel: 'Oldest run', sortKey: 'endTimestamp', columnIndex: 4 } ]; const resourceName = { From 8f783e1fb0cb29061f297e34d8a13aad09f59ae2 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 11 Apr 2024 12:32:00 +0530 Subject: [PATCH 06/13] made changes to filter test urls on their access_type --- .../main/java/com/akto/testing_cli/Main.java | 3 ++- .../main/java/com/akto/test_editor/Utils.java | 14 +++++++++++ .../akto/test_editor/filter/FilterAction.java | 23 ++++++++++++++++++- .../akto/dao/test_editor/TestEditorEnums.java | 3 ++- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/apps/testing-cli/src/main/java/com/akto/testing_cli/Main.java b/apps/testing-cli/src/main/java/com/akto/testing_cli/Main.java index 929da3815e..5ec48a5c11 100644 --- a/apps/testing-cli/src/main/java/com/akto/testing_cli/Main.java +++ b/apps/testing-cli/src/main/java/com/akto/testing_cli/Main.java @@ -182,7 +182,8 @@ public static void main(String[] args) { || content.contains(ContextOperator.PARAM_CONTEXT.toString().toLowerCase()) || content.contains(ContextOperator.PRIVATE_VARIABLE_CONTEXT.toString().toLowerCase()) || content.contains(ContextOperator.INCLUDE_ROLES_ACCESS.toString().toLowerCase()) - || content.contains(ContextOperator.EXCLUDE_ROLES_ACCESS.toString().toLowerCase())){ + || content.contains(ContextOperator.EXCLUDE_ROLES_ACCESS.toString().toLowerCase()) + || content.contains(ContextOperator.API_ACCESS_TYPE.toString().toLowerCase())){ String info = "Cannot run context tests. Skipping " + obj; logger.info(info); return false; diff --git a/apps/testing/src/main/java/com/akto/test_editor/Utils.java b/apps/testing/src/main/java/com/akto/test_editor/Utils.java index f540651c78..09b97ef3ae 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/Utils.java +++ b/apps/testing/src/main/java/com/akto/test_editor/Utils.java @@ -18,6 +18,7 @@ import com.akto.dto.OriginalHttpRequest; import com.akto.dto.RawApi; +import com.akto.dto.ApiInfo.ApiAccessType; import com.akto.dto.test_editor.ExecutorSingleOperationResp; import com.akto.dto.testing.UrlModifierPayload; import com.akto.util.Constants; @@ -744,4 +745,17 @@ public static Boolean sendRequestToSsrfServer(String url){ } } + public static ApiAccessType getApiAccessTypeFromString(String apiAccessType){ + switch (apiAccessType) { + case "private": + return ApiAccessType.PRIVATE; + case "public": + return ApiAccessType.PUBLIC; + case "partner": + return ApiAccessType.PARTNER; + default: + return null; + } + } + } diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java b/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java index a1797274a8..0b524194bc 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java +++ b/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java @@ -13,6 +13,7 @@ import org.bson.conversions.Bson; +import com.akto.dao.ApiInfoDao; import com.akto.dao.SampleDataDao; import com.akto.dao.SingleTypeInfoDao; import com.akto.dao.test_editor.TestEditorEnums; @@ -22,6 +23,7 @@ import com.akto.dto.HttpResponseParams; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.RawApi; +import com.akto.dto.ApiInfo.ApiAccessType; import com.akto.dto.test_editor.DataOperandFilterRequest; import com.akto.dto.test_editor.DataOperandsFilterResponse; import com.akto.dto.test_editor.FilterActionRequest; @@ -81,7 +83,9 @@ public DataOperandsFilterResponse evaluateContext(FilterActionRequest filterActi case "include_roles_access": return evaluateRolesAccessContext(filterActionRequest, true); case "exclude_roles_access": - return evaluateRolesAccessContext(filterActionRequest, false); + return evaluateRolesAccessContext(filterActionRequest, false); + case "api_access_type": + return applyFilterOnAccessType(filterActionRequest); default: return new DataOperandsFilterResponse(false, null, null, null); } @@ -1185,6 +1189,23 @@ private DataOperandsFilterResponse evaluateRolesAccessContext(FilterActionReques return new DataOperandsFilterResponse(res, null, null, null); } + private DataOperandsFilterResponse applyFilterOnAccessType(FilterActionRequest filterActionRequest){ + List querySet = (List) filterActionRequest.getQuerySet(); + ApiInfo.ApiInfoKey apiInfoKey = filterActionRequest.getApiInfoKey(); + ApiInfo apiInfo = ApiInfoDao.instance.findOne(ApiInfoDao.getFilter(apiInfoKey)); + Set apiAccessTypes = apiInfo.getApiAccessTypes(); + boolean res = false; + if(apiInfo != null && !querySet.isEmpty() && apiAccessTypes.size() > 0){ + ApiAccessType apiAccessType = Utils.getApiAccessTypeFromString(querySet.get(0).toString()); + if(apiAccessTypes.size() == 1){ + res = apiAccessTypes.contains(apiAccessType); + }else{ + res = apiAccessType == ApiAccessType.PUBLIC; + } + } + return new DataOperandsFilterResponse(res, null, null, null); + } + public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpRequest, ApiInfo.ApiInfoKey apiInfoKey) { String urlWithParams = originalHttpRequest.getFullUrlWithParams(); String url = apiInfoKey.url; diff --git a/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java b/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java index 227c02359e..083b00231a 100644 --- a/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java +++ b/libs/dao/src/main/java/com/akto/dao/test_editor/TestEditorEnums.java @@ -76,7 +76,8 @@ public enum ContextOperator { PARAM_CONTEXT, ENDPOINT_IN_TRAFFIC_CONTEXT, INCLUDE_ROLES_ACCESS, - EXCLUDE_ROLES_ACCESS + EXCLUDE_ROLES_ACCESS, + API_ACCESS_TYPE } From cb3ece845741e1f049013ae3154a9e7210c12bf3 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 11 Apr 2024 12:38:20 +0530 Subject: [PATCH 07/13] fixed check for access_type --- apps/testing/src/main/java/com/akto/test_editor/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/testing/src/main/java/com/akto/test_editor/Utils.java b/apps/testing/src/main/java/com/akto/test_editor/Utils.java index 09b97ef3ae..fb494d8ae3 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/Utils.java +++ b/apps/testing/src/main/java/com/akto/test_editor/Utils.java @@ -746,7 +746,7 @@ public static Boolean sendRequestToSsrfServer(String url){ } public static ApiAccessType getApiAccessTypeFromString(String apiAccessType){ - switch (apiAccessType) { + switch (apiAccessType.toLowerCase()) { case "private": return ApiAccessType.PRIVATE; case "public": From 5efdf9e910008042c0db476db1141b77cc5f8c19 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 12 Apr 2024 14:43:16 +0530 Subject: [PATCH 08/13] made changes to implement exact numbers of content after filtering --- .../components/tables/GithubServerTable.js | 26 +++++-- .../components/tables/TableContext.js | 35 ++++++++++ .../components/tables/tableReducer.js | 21 ++++++ .../dashboard/components/tables/transform.js | 4 +- .../api_collections/ApiCollections.jsx | 58 ++++++---------- .../observe/api_collections/ApiEndpoints.jsx | 68 ++++++------------- .../component/ApiChangesTable.jsx | 37 ++++------ .../api_collections/data/apiChanges.js | 2 +- .../SingleTestRunPage/SingleTestRunPage.js | 39 +++-------- .../testing/TestRunsPage/TestRunsPage.js | 43 +++--------- .../web/polaris_web/web/src/apps/main/App.js | 3 + .../web/src/apps/main/PersistStore.js | 2 + .../web/polaris_web/web/src/util/func.js | 30 ++++++++ 13 files changed, 187 insertions(+), 181 deletions(-) create mode 100644 apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/TableContext.js create mode 100644 apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/tableReducer.js diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js index eb019eb931..d6e3d0efe1 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js @@ -21,12 +21,18 @@ import transform from '../../pages/observe/transform'; import DropdownSearch from '../shared/DropdownSearch'; import PersistStore from '../../../main/PersistStore'; import tableFunc from './transform'; +import useTable from './TableContext'; function GithubServerTable(props) { const filtersMap = PersistStore(state => state.filtersMap) const setFiltersMap = PersistStore(state => state.setFiltersMap) - const pageFiltersMap = filtersMap[window.location.href] + const tableInitialState = PersistStore(state => state.tableInitialState) + const setTableInitialState = PersistStore(state => state.setTableInitialState) + + const currentPageKey = window.location.href + + const pageFiltersMap = filtersMap[currentPageKey] const initialStateFilters = tableFunc.mergeFilters(props.appliedFilters || [], (pageFiltersMap?.filters || []),props.disambiguateLabel) const { mode, setMode } = useSetIndexFiltersMode(props?.mode ? props.mode : IndexFiltersMode.Filtering); const [sortSelected, setSortSelected] = useState(tableFunc.getInitialSortSelected(props.sortOptions, pageFiltersMap)) @@ -37,6 +43,8 @@ function GithubServerTable(props) { const [appliedFilters, setAppliedFilters] = useState(initialStateFilters); const [queryValue, setQueryValue] = useState(''); + const { applyFilter, tabsInfo } = useTable() + const [sortableColumns, setSortableColumns] = useState([]) const [activeColumnSort, setActiveColumnSort] = useState({columnIndex: -1, sortDirection: 'descending'}) @@ -48,7 +56,7 @@ function GithubServerTable(props) { const tableTabs = props.tableTabs ? props.tableTabs : props.tabs if(tableTabs){ const primitivePath = window.location.origin + window.location.pathname - const newUrl = primitivePath + "#" + tableTabs[x].content + const newUrl = primitivePath + "#" + tableTabs[x].id window.history.replaceState(null, null, newUrl) } } @@ -56,7 +64,7 @@ function GithubServerTable(props) { useEffect(()=> { setAppliedFilters(initialStateFilters) setSortSelected(tableFunc.getInitialSortSelected(props.sortOptions, pageFiltersMap)) - },[window.location.href]) + },[currentPageKey]) useEffect(() => { let [sortKey, sortOrder] = sortSelected.length == 0 ? ["", ""] : sortSelected[0].split(" "); @@ -69,6 +77,12 @@ function GithubServerTable(props) { let tempData = await props.fetchData(sortKey, sortOrder == 'asc' ? -1 : 1, page * pageLimit, pageLimit, filters, filterOperators, queryValue); tempData ? setData([...tempData.value]) : setData([]) tempData ? setTotal(tempData.total) : setTotal(0) + applyFilter(tempData.total) + + setTableInitialState({ + ...tableInitialState, + [currentPageKey]: tempData.total + }) } handleSelectedTab(props?.selected) fetchData(); @@ -88,7 +102,7 @@ function GithubServerTable(props) { setSortSelected([tempSortSelected[0].value]) } let copyFilters = filtersMap - copyFilters[window.location.href] = { + copyFilters[currentPageKey] = { 'filters': pageFiltersMap?.filters || [], 'sort': sortVal } @@ -110,7 +124,7 @@ function GithubServerTable(props) { } else { setAppliedFilters(temp); let tempFilters = filtersMap - tempFilters[window.location.href] = { + tempFilters[currentPageKey] = { 'filters': temp, 'sort': pageFiltersMap?.sort || [] } @@ -133,7 +147,7 @@ function GithubServerTable(props) { } setPage(0); let tempFilters = filtersMap - tempFilters[window.location.href] = { + tempFilters[currentPageKey] = { 'filters': temp, 'sort': pageFiltersMap?.sort || [] } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/TableContext.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/TableContext.js new file mode 100644 index 0000000000..d834114466 --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/TableContext.js @@ -0,0 +1,35 @@ +import { createContext, useReducer, useContext } from "react"; +import tableReducer, {initialState} from "./tableReducer"; + +const TableContext = createContext(initialState); + +export const TableContextProvider = ({ children }) => { + const [state, dispatch] = useReducer(tableReducer, initialState); + + const applyFilter = (filter) => { + dispatch({ + type: "APPLY_FILTER", + payload: { + tabsInfo: filter + } + }); + }; + + const value = { + tabsInfo: state.tabsInfo, + applyFilter, + }; + return {children}; +}; + +const useTable = () => { + const context = useContext(TableContext); + + if (context === undefined) { + throw new Error("useTable must be used within TableContext"); + } + + return context; +}; + +export default useTable; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/tableReducer.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/tableReducer.js new file mode 100644 index 0000000000..d0c40d9f3f --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/tableReducer.js @@ -0,0 +1,21 @@ +import PersistStore from "../../../main/PersistStore"; +const tableInitialState = PersistStore.getState().tableInitialState[window.location.href] || {} + +export const initialState = { + tabsInfo : tableInitialState +} + +const tableReducer = (state, action) =>{ + const { type, payload } = action; + switch (type) { + case "APPLY_FILTER": + return { + ...state, + tabsInfo: payload.tabsInfo, + }; + default: + return{...state} + } +} + +export default tableReducer diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/transform.js index 94eecca357..d6f2e17e26 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/transform.js @@ -9,7 +9,7 @@ const tableFunc = { let key = header.filterKey || header.value let label = header.filterLabel || header.text let allItemValues = [] - props.data.forEach(i => { + props.data && props.data.forEach(i => { let value = i[key] if (value instanceof Set) { allItemValues = allItemValues.concat(...value) @@ -127,7 +127,7 @@ const tableFunc = { }, getInitialSortSelected(sortOptions, filtersMap){ if(!sortOptions || sortOptions === undefined || sortOptions.length === 0){ - return {columnIndex: -1, sortDirection: 'descending'} + return [''] } if(!filtersMap || filtersMap?.sort === undefined || filtersMap.sort.length === 0){ return [sortOptions[0].value] diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx index 5640847d14..ed35f82acf 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx @@ -3,17 +3,19 @@ import { Text, Button, IndexFiltersMode, Box, Badge, Popover, ActionList } from import api from "../api" import { useEffect,useState, useRef } from "react" import func from "@/util/func" -import GithubSimpleTable from "../../../components/tables/GithubSimpleTable"; +import GithubSimpleTable from "@/apps/dashboard/components/tables/GithubSimpleTable"; import { CircleTickMajor } from '@shopify/polaris-icons'; import ObserveStore from "../observeStore" import PersistStore from "../../../../main/PersistStore" import transform from "../transform" -import SpinnerCentered from "../../../components/progress/SpinnerCentered" -import { CellType } from "../../../components/tables/rows/GithubRow" +import SpinnerCentered from "@/apps/dashboard/components/progress/SpinnerCentered" +import { CellType } from "@/apps/dashboard/components/tables/rows/GithubRow" import CreateNewCollectionModal from "./CreateNewCollectionModal" -import TooltipText from "../../../components/shared/TooltipText" -import SummaryCardInfo from "../../../components/shared/SummaryCardInfo" +import TooltipText from "@/apps/dashboard/components/shared/TooltipText" +import SummaryCardInfo from "@/apps/dashboard/components/shared/SummaryCardInfo" import CollectionsPageBanner from "./component/CollectionsPageBanner" +import useTable from "@/apps/dashboard/components/tables/TableContext" + const headers = [ { @@ -122,44 +124,22 @@ const convertToNewData = (collectionsArr, sensitiveInfoMap, severityInfoMap, cov function ApiCollections() { - const [data, setData] = useState({'All':[]}) + const [data, setData] = useState({'all':[]}) const [active, setActive] = useState(false); const [loading, setLoading] = useState(false) - const [selectedTab, setSelectedTab] = useState("All") + const [selectedTab, setSelectedTab] = useState("all") const [selected, setSelected] = useState(0) const [summaryData, setSummaryData] = useState({totalEndpoints:0 , totalTestedEndpoints: 0, totalSensitiveEndpoints: 0, totalCriticalEndpoints: 0}) const [hasUsageEndpoints, setHasUsageEndpoints] = useState(false) const [envTypeMap, setEnvTypeMap] = useState({}) const [refreshData, setRefreshData] = useState(false) const [popover,setPopover] = useState(false) - - - const tableTabs = [ - { - content: 'All', - badge: data["All"]?.length?.toString(), - onAction: () => { setSelectedTab('All') }, - id: 'All', - }, - { - content: 'Hostname', - badge: data["Hostname"]?.length?.toString(), - onAction: () => { setSelectedTab('Hostname') }, - id: 'Hostname', - }, - { - content: 'Groups', - badge: data["Groups"]?.length?.toString(), - onAction: () => { setSelectedTab('Groups') }, - id: 'Groups', - }, - { - content: 'Custom', - badge: data["Custom"]?.length?.toString(), - onAction: () => { setSelectedTab('Custom') }, - id: 'Custom', - } - ] + + const definedTableTabs = ['All', 'Hostname', 'Groups', 'Custom'] + + const { tabsInfo } = useTable() + const tableCountObj = func.getTabsCount(definedTableTabs, data) + const tableTabs = func.getTableTabsContent(definedTableTabs, tableCountObj, setSelectedTab, selectedTab, tabsInfo) const setInventoryFlyout = ObserveStore(state => state.setInventoryFlyout) const setFilteredItems = ObserveStore(state => state.setFilteredItems) @@ -227,10 +207,10 @@ function ApiCollections() { setHostNameMap(allHostNameMap) tmp = {} - tmp.All = dataObj.prettify - tmp.Hostname = dataObj.prettify.filter((c) => c.hostName !== null && c.hostName !== undefined) - tmp.Groups = dataObj.prettify.filter((c) => c.type === "API_GROUP") - tmp.Custom = tmp.All.filter(x => !tmp.Hostname.includes(x) && !tmp.Groups.includes(x)); + tmp.all = dataObj.prettify + tmp.hostname = dataObj.prettify.filter((c) => c.hostName !== null && c.hostName !== undefined) + tmp.groups = dataObj.prettify.filter((c) => c.type === "API_GROUP") + tmp.custom = tmp.all.filter(x => !tmp.hostname.includes(x) && !tmp.groups.includes(x)); setData(tmp); } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index f21c48caa4..348669cd1b 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -25,6 +25,7 @@ import TooltipText from "../../../components/shared/TooltipText" import EmptyScreensLayout from "../../../components/banners/EmptyScreensLayout" import { ENDPOINTS_PAGE_DOCS_URL } from "../../../../main/onboardingData" import {TestrunsBannerComponent} from "../../testing/TestRunsPage/TestrunsBannerComponent" +import useTable from "../../../components/tables/TableContext" const headings = [ { @@ -127,7 +128,7 @@ function ApiEndpoints() { const [runTests, setRunTests ] = useState(false) const [endpointData, setEndpointData] = useState([]) - const [selectedTab, setSelectedTab] = useState("All") + const [selectedTab, setSelectedTab] = useState("all") const [selected, setSelected] = useState(0) const [loading, setLoading] = useState(true) const [apiDetail, setApiDetail] = useState({}) @@ -146,43 +147,12 @@ function ApiEndpoints() { const queryParams = new URLSearchParams(location.search); const selectedUrl = queryParams.get('selected_url') const selectedMethod = queryParams.get('selected_method') - const tableTabs = [ - { - content: 'All', - index: 0, - badge: endpointData["All"]?.length?.toString(), - onAction: ()=> {setSelectedTab('All')}, - id: 'All', - }, - { - content: 'New', - index: 1, - badge: endpointData["New"]?.length?.toString(), - onAction: ()=> {setSelectedTab('New')}, - id: 'New', - }, - { - content: 'Sensitive', - index: 2, - badge: endpointData["Sensitive"]?.length?.toString(), - onAction: ()=> {setSelectedTab('Sensitive')}, - id:'Sensitive', - }, - { - content: 'High risk', - index: 3, - badge: endpointData["Risk"]?.length?.toString(), - onAction: ()=> {setSelectedTab('Risk')}, - id: 'Risk', - }, - { - content: 'No auth detected', - index: 4, - badge: endpointData["No_auth"]?.length?.toString(), - onAction: ()=> {setSelectedTab('No_auth')}, - id: 'No_auth' - }, - ] + + const definedTableTabs = ['All', 'New', 'Sensitive', 'High risk', 'No auth'] + + const { tabsInfo } = useTable() + const tableCountObj = func.getTabsCount(definedTableTabs, endpointData) + const tableTabs = func.getTableTabsContent(definedTableTabs, tableCountObj, setSelectedTab, selectedTab, tabsInfo) async function fetchData() { setLoading(true) @@ -214,13 +184,13 @@ function ApiEndpoints() { let data = {} let allEndpoints = func.mergeApiInfoAndApiCollection(apiEndpointsInCollection, apiInfoListInCollection, null) const prettifyData = transform.prettifyEndpointsData(allEndpoints) - data['All'] = prettifyData - data['Sensitive'] = prettifyData.filter(x => x.sensitive && x.sensitive.size > 0) - data['Risk'] = prettifyData.filter(x=> x.riskScore >= 4) - data['New'] = prettifyData.filter(x=> x.isNew) - data['No_auth'] = prettifyData.filter(x => x.open) + data['all'] = prettifyData + data['sensitive'] = prettifyData.filter(x => x.sensitive && x.sensitive.size > 0) + data['high_risk'] = prettifyData.filter(x=> x.riskScore >= 4) + data['new'] = prettifyData.filter(x=> x.isNew) + data['no_auth'] = prettifyData.filter(x => x.open) setEndpointData(data) - setSelectedTab("All") + setSelectedTab("all") setSelected(0) setApiEndpoints(apiEndpointsInCollection) @@ -231,8 +201,8 @@ function ApiEndpoints() { } useEffect(() => { - if (!endpointData || !endpointData["All"] || !selectedUrl || !selectedMethod) return - let allData = endpointData["All"] + if (!endpointData || !endpointData["all"] || !selectedUrl || !selectedMethod) return + let allData = endpointData["all"] const selectedApiDetail = allData.filter((x) => { return selectedUrl === x.endpoint && selectedMethod === x.method @@ -336,7 +306,7 @@ function ApiEndpoints() { let headerTextToValueMap = Object.fromEntries(headers.map(x => [x.text, x.type === CellType.TEXT ? x.value : x.textValue]).filter(x => x[0].length > 0)); let csv = Object.keys(headerTextToValueMap).join(",") + "\r\n" - const allEndpoints = endpointData['All'] + const allEndpoints = endpointData['all'] allEndpoints.forEach(i => { csv += Object.values(headerTextToValueMap).map(h => (i[h] || "-")).join(",") + "\r\n" }) @@ -517,7 +487,7 @@ function ApiEndpoints() { @@ -593,7 +563,7 @@ function ApiEndpoints() { ] : showEmptyScreen ? [ state.dataTypeNames); const apiCollectionMap = PersistStore(state => state.collectionsMap) const [loading, setLoading] = useState(false); const [filters, setFilters] = useState([]) - const tableTabs = [ - { - content: 'New endpoints', - index: 0, - badge: transform.formatNumberWithCommas(newEndpoints.length), - onAction: ()=> {setSelectedTab('endpoints')}, - id: 'endpoints', - }, - { - content: 'New parameters', - index: 1, - badge: transform.formatNumberWithCommas(parametersCount), - onAction: ()=> {setSelectedTab('param')}, - id: 'param', - }, - ] + const definedTableTabs = ['New endpoints', 'New params'] + const initialCount = [0 , parametersCount] + + const { tabsInfo } = useTable() + const tableCountObj = func.getTabsCount(definedTableTabs, newEndpoints, initialCount) + const tableTabs = func.getTableTabsContent(definedTableTabs, tableCountObj, setSelectedTab, selectedTab, tabsInfo) const tableDataObj = apiChangesData.getData(selectedTab); const handleRow = (data) => { let headers = [] - if(selectedTab === 'param'){ + if(selectedTab.includes('param')){ headers = transform.getParamHeaders() ; }else{ headers = transform.getDetailsHeaders() ; @@ -65,7 +56,7 @@ function ApiChangesTable(props) { }) function disambiguateLabel(key, value) { - if(selectedTab === 'param'){ + if(selectedTab.includes('param')){ switch (key) { case "apiCollectionId": return func.convertToDisambiguateLabelObj(value, apiCollectionMap, 3) @@ -86,7 +77,7 @@ function ApiChangesTable(props) { } const fetchTableData = async(sortKey, sortOrder, skip, limit, filters, filterOperators, queryValue) =>{ - if(selectedTab === 'param'){ + if(selectedTab.includes('param')){ setLoading(true); let ret = []; let total = 0; @@ -118,8 +109,8 @@ function ApiChangesTable(props) { loading={loading || tableLoading} onRowClick={(data) => handleRow(data)} fetchData={fetchTableData} - filters={selectedTab === 'param' ? paramFilters : filters} - hideQueryField={selectedTab === 'param' ? true : false} + filters={selectedTab.includes('param') ? paramFilters : filters} + hideQueryField={selectedTab.includes('param') ? true : false} selected={selected} onSelect={handleSelectedTab} mode={IndexFiltersMode.Default} diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/data/apiChanges.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/data/apiChanges.js index 04e6eb6f49..d2427e0447 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/data/apiChanges.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/data/apiChanges.js @@ -194,7 +194,7 @@ let paramFilters = [ const apiChangesData = { getData(key){ - if(key === 'param'){ + if(key.includes('param')){ const obj = { headers: [...newParametersHeaders, ...methodObj], headings: newParametersHeaders, diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js index 3ac52f96fd..40b5d029dd 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js @@ -28,6 +28,7 @@ import TooltipText from "../../../components/shared/TooltipText"; import PersistStore from "../../../../main/PersistStore"; import TrendChart from "./TrendChart"; import { CellType } from "../../../components/tables/rows/GithubRow"; +import useTable from "../../../components/tables/TableContext"; let headers = [ { @@ -126,13 +127,13 @@ let filters = [ function SingleTestRunPage() { - const [testRunResults, setTestRunResults] = useState({ vulnerable: [], secured: [], skipped: [] }) + const [testRunResults, setTestRunResults] = useState({ vulnerable: [], no_vulnerability_found: [], skipped: [] }) const [ selectedTestRun, setSelectedTestRun ] = useState({}); const subCategoryFromSourceConfigMap = PersistStore(state => state.subCategoryFromSourceConfigMap); const subCategoryMap = PersistStore(state => state.subCategoryMap); const params= useParams() const [loading, setLoading] = useState(false); - const [tempLoading , setTempLoading] = useState({vulnerable: false, secured: false, skipped: false, running: false}) + const [tempLoading , setTempLoading] = useState({vulnerable: false, no_vulnerability_found: false, skipped: false, running: false}) const [selectedTab, setSelectedTab] = useState("vulnerable") const [selected, setSelected] = useState(0) const [workflowTest, setWorkflowTest ] = useState(false); @@ -170,7 +171,7 @@ function SingleTestRunPage() { setLoading(false); setTempLoading((prev) => { prev.vulnerable = true; - prev.secured = true; + prev.no_vulnerability_found = true; prev.skipped = true; return {...prev}; }); @@ -190,7 +191,7 @@ function SingleTestRunPage() { await api.fetchTestingRunResults(summaryHexId, "SECURED").then(({ testingRunResults }) => { testRunResults = transform.prepareTestRunResults(hexId, testingRunResults, subCategoryMap, subCategoryFromSourceConfigMap) }) - fillData(transform.getPrettifiedTestRunResults(testRunResults), 'secured') + fillData(transform.getPrettifiedTestRunResults(testRunResults), 'no_vulnerability_found') } async function fetchData(setData) { @@ -313,29 +314,11 @@ const promotedBulkActions = (selectedDataHexIds) => { } } - const tableTabs = [ - { - content: 'Vulnerable', - index: 0, - badge: testRunResults["vulnerable"]?.length?.toString(), - onAction: ()=> {setSelectedTab('vulnerable')}, - id: 'vulnerable', - }, - { - content: 'Skipped', - index: 1, - badge: testRunResults["skipped"]?.length?.toString(), - onAction: ()=> {setSelectedTab('skipped')}, - id: 'skipped', - }, - { - content: 'No vulnerability found', - index: 2, - badge: testRunResults["secured"]?.length?.toString(), - onAction: ()=> {setSelectedTab('secured')}, - id: 'secured', - } - ] + const definedTableTabs = ['Vulnerable', 'Skipped', 'No vulnerability found'] + + const { tabsInfo } = useTable() + const tableCountObj = func.getTabsCount(definedTableTabs, testRunResults) + const tableTabs = func.getTableTabsContent(definedTableTabs, tableCountObj, setSelectedTab, selectedTab, tabsInfo) const handleSelectedTab = (selectedIndex) => { setLoading(true) @@ -449,7 +432,7 @@ const promotedBulkActions = (selectedDataHexIds) => { ) } - const allResultsLength = testRunResults.skipped.length + testRunResults.secured.length + testRunResults.vulnerable.length + const allResultsLength = testRunResults.skipped.length + testRunResults.no_vulnerability_found.length + testRunResults.vulnerable.length const useComponents = (!workflowTest && allResultsLength === 0) ? [] : components return ( diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestRunsPage.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestRunsPage.js index 01b5af7574..0960f3b5ee 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestRunsPage.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/TestRunsPage/TestRunsPage.js @@ -19,6 +19,7 @@ import DateRangeFilter from "../../../components/layouts/DateRangeFilter"; import {produce} from "immer" import values from "@/util/values"; import {TestrunsBannerComponent} from "./TestrunsBannerComponent"; +import useTable from "../../../components/tables/TableContext"; /* { @@ -191,7 +192,7 @@ const endTimestamp = getTimeEpoch("until") + 86400 const [loading, setLoading] = useState(true); -const [currentTab, setCurrentTab] = useState("oneTime"); +const [currentTab, setCurrentTab] = useState("one_time"); const [updateTable, setUpdateTable] = useState(false); const [countMap, setCountMap] = useState({}); const [selected, setSelected] = useState(1); @@ -237,7 +238,7 @@ function processData(testingRuns, latestTestingRunResultSummaries, cicd){ switch (currentTab) { - case "cicd": + case "ci_cd": await api.fetchTestingDetails( startTimestamp, endTimestamp, sortKey, sortOrder, skip, limit, filters, "CI_CD", ).then(({ testingRuns, testingRunsCount, latestTestingRunResultSummaries }) => { @@ -253,7 +254,7 @@ function processData(testingRuns, latestTestingRunResultSummaries, cicd){ total = testingRunsCount; }); break; - case "oneTime": + case "one_time": await api.fetchTestingDetails( startTimestamp, endTimestamp, sortKey, sortOrder, skip, limit, filters, "ONE_TIME" ).then(({ testingRuns, testingRunsCount, latestTestingRunResultSummaries }) => { @@ -305,36 +306,12 @@ function processData(testingRuns, latestTestingRunResultSummaries, cicd){ }) } - const tableTabs = [ - { - content: 'All', - index: 0, - badge: countMap['allTestRuns']?.toString(), - onAction: ()=> {setCurrentTab('All')}, - id: 'All', - }, - { - content: 'One time', - index: 0, - badge: countMap['oneTime']?.toString(), - onAction: ()=> {setCurrentTab('oneTime')}, - id: 'oneTime', - }, - { - content: 'Recurring', - index: 0, - badge: countMap['scheduled']?.toString(), - onAction: ()=> {setCurrentTab('scheduled')}, - id: 'scheduled', - }, - { - content: 'CI/CD', - index: 0, - badge: countMap['cicd']?.toString(), - onAction: ()=> {setCurrentTab('cicd')}, - id: 'cicd', - }, - ] + const definedTableTabs = ['All', 'One time', 'Scheduled', 'CI/CD'] + const initialCount = [countMap['allTestRuns'], countMap['ontTime'], countMap['scheduled'], countMap['cicd']] + + const { tabsInfo } = useTable() + const tableCountObj = func.getTabsCount(definedTableTabs, {}, initialCount) + const tableTabs = func.getTableTabsContent(definedTableTabs, tableCountObj, setCurrentTab, currentTab, tabsInfo) const fetchTotalCount = () =>{ setLoading(true) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/main/App.js b/apps/dashboard/web/polaris_web/web/src/apps/main/App.js index 104df1c300..d9b621f453 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/main/App.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/main/App.js @@ -66,6 +66,7 @@ import SignupPage from "../signup/pages/SignupPage"; import PageCheckInbox from "../signup/pages/PageCheckInbox" import PageBusinessEmail from "../signup/pages/PageBusinessEmail" import TokenValidator from "./TokenValidator" +import { TableContextProvider } from "@/apps/dashboard/components/tables/TableContext"; // if you add a component in a new path, please verify the search implementation in function -> 'getSearchItemsArr' in func.js @@ -353,7 +354,9 @@ function App() { }, []) return ( + + ); } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/main/PersistStore.js b/apps/dashboard/web/polaris_web/web/src/apps/main/PersistStore.js index f7bf0598d9..704346dd42 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/main/PersistStore.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/main/PersistStore.js @@ -23,6 +23,7 @@ const initialState = { // selectedSampleApi: {}, coverageMap:{}, filtersMap:{}, + tableInitialState: {}, }; let persistStore = (set) => ({ @@ -44,6 +45,7 @@ let persistStore = (set) => ({ // setSelectedSampleApi: (selectedSampleApi) => set({selectedSampleApi: selectedSampleApi}), setCoverageMap:(coverageMap)=>{set({coverageMap: coverageMap})}, setFiltersMap: (filtersMap) => set({ filtersMap }), + setTableInitialState: (tableInitialState) => set({ tableInitialState }), resetAll: () => set(initialState), // Reset function }) diff --git a/apps/dashboard/web/polaris_web/web/src/util/func.js b/apps/dashboard/web/polaris_web/web/src/util/func.js index 377a816fcf..17b04db495 100644 --- a/apps/dashboard/web/polaris_web/web/src/util/func.js +++ b/apps/dashboard/web/polaris_web/web/src/util/func.js @@ -1369,6 +1369,36 @@ mapCollectionIdToHostName(apiCollections){ } return hash; }, + getTableTabsContent(tableTabs, countObj, setSelectedTab, selectedTab, currentCount){ + const finalTabs = tableTabs.map((tab,ind) => { + const tabId = this.getKeyFromName(tab) + return { + content: tab, + badge: selectedTab === tabId ? currentCount.toString() : countObj[tabId].toString(), + onAction: () => { setSelectedTab(tabId) }, + id: this.getKeyFromName(tabId), + index: ind + } + }) + return finalTabs + }, + getTabsCount(tableTabs, data, initialCountArr = []){ + const currentState = PersistStore(state => state.tableInitialState) + const baseUrl = window.location.href.split('#')[0] + + let finalCountObj = {} + tableTabs.forEach((tab,ind) => { + const tabId = this.getKeyFromName(tab) + const tabKey = baseUrl + '#' + tabId + const count = currentState[tabKey] || data[tabId]?.length || initialCountArr[ind] || 0 + finalCountObj[tabId] = count + }) + + return finalCountObj + }, + getKeyFromName(key){ + return key.replace(/[\s/]+/g, '_').toLowerCase(); + } } export default func \ No newline at end of file From 58d0f532f0761f5c31687e599c3dbe86bde454d3 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 12 Apr 2024 17:29:09 +0530 Subject: [PATCH 09/13] fixing timestamp filter and extracting timefilter for sensitive data --- .../components/tables/GithubServerTable.js | 58 +++++-------------- .../SensitiveDataExposure.jsx | 38 +++++++----- 2 files changed, 36 insertions(+), 60 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js index d6e3d0efe1..68ed1b3c11 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/tables/GithubServerTable.js @@ -11,12 +11,8 @@ import { ChoiceList, Tabs} from '@shopify/polaris'; import {GithubRow} from './rows/GithubRow'; -import { useState, useCallback, useEffect, useReducer } from 'react'; -import DateRangePicker from '../layouts/DateRangePicker'; +import { useState, useCallback, useEffect } from 'react'; import "./style.css" -import func from '@/util/func'; -import { produce } from "immer" -import values from "@/util/values" import transform from '../../pages/observe/transform'; import DropdownSearch from '../shared/DropdownSearch'; import PersistStore from '../../../main/PersistStore'; @@ -43,15 +39,13 @@ function GithubServerTable(props) { const [appliedFilters, setAppliedFilters] = useState(initialStateFilters); const [queryValue, setQueryValue] = useState(''); - const { applyFilter, tabsInfo } = useTable() + const { applyFilter } = useTable() const [sortableColumns, setSortableColumns] = useState([]) const [activeColumnSort, setActiveColumnSort] = useState({columnIndex: -1, sortDirection: 'descending'}) let filterOperators = props.headers.reduce((map, e) => { map[e.sortKey || e.value] = 'OR'; return map }, {}) - const [currDateRange, dispatchCurrDateRange] = useReducer(produce((draft, action) => func.dateRangeReducer(draft, action)), values.ranges[3]); - const handleSelectedTab = (x) => { const tableTabs = props.tableTabs ? props.tableTabs : props.tabs if(tableTabs){ @@ -119,17 +113,13 @@ function GithubServerTable(props) { temp.push(defaultAppliedFilter) } }) - if (key === "dateRange") { - getDate({ type: "update", period: values.ranges[3] }); - } else { - setAppliedFilters(temp); - let tempFilters = filtersMap - tempFilters[currentPageKey] = { - 'filters': temp, - 'sort': pageFiltersMap?.sort || [] - } - setFiltersMap(tempFilters) + setAppliedFilters(temp); + let tempFilters = filtersMap + tempFilters[currentPageKey] = { + 'filters': temp, + 'sort': pageFiltersMap?.sort || [] } + setFiltersMap(tempFilters) } const changeAppliedFilters = (key, value) => { @@ -168,12 +158,6 @@ function GithubServerTable(props) { changeAppliedFilters(key, value); } - const getDate = (dateObj) => { - dispatchCurrDateRange({type: "update", period: dateObj.period, title: dateObj.title, alias: dateObj.alias}) - let obj = dateObj.period.period; - handleFilterStatusChange("dateRange",obj) - } - const getSortedChoices = (choices) => { return choices.sort((a, b) => (a?.label || a) - (b?.label || b)); } @@ -221,28 +205,14 @@ function GithubServerTable(props) { } }) } - if (props.calenderFilter) { - filters.push({ - key: "dateRange", - label: props.calenderLabel || "Discovered", - filter: - ( - getDate(dateObj)} - setPopoverState={() => {}} - />), - pinned: true - }) - } const handleFiltersClearAll = useCallback(() => { - let tempFilters = pageFiltersMap - delete tempFilters.filters - setFiltersMap(tempFilters) - if(appliedFilters.length > 0 && appliedFilters.filter((x) => x.key === 'dateRange')){ - getDate({ type: "update", period: values.ranges[3] }); - } + setFiltersMap({ + ...filtersMap, + [currentPageKey]:{ + sort: filtersMap[currentPageKey]?.sort || [] + } + }) setAppliedFilters([]) }, []); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx index c0e60c7af5..2da740efaa 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx @@ -1,9 +1,10 @@ -import { Text, Button } from "@shopify/polaris" +import { Text, Button, HorizontalStack } from "@shopify/polaris" import PageWithMultipleCards from "../../../components/layouts/PageWithMultipleCards" import GithubServerTable from "../../../components/tables/GithubServerTable" -import { useEffect, useState } from "react" +import { useReducer, useState } from "react" +import values from "@/util/values"; +import {produce} from "immer" import api from "../api" -import Store from "../../../store" import func from "@/util/func" import { useNavigate, useParams } from "react-router-dom" import { @@ -11,6 +12,7 @@ import { DynamicSourceMinor, ClockMinor } from '@shopify/polaris-icons'; import StyledEndpoint from "../api_collections/component/StyledEndpoint" import PersistStore from "../../../../main/PersistStore" +import DateRangeFilter from "../../../components/layouts/DateRangeFilter" const headers = [ { @@ -135,6 +137,14 @@ function SensitiveDataExposure() { const subType = params.subType; const apiCollectionMap = PersistStore(state => state.collectionsMap) + const [currDateRange, dispatchCurrDateRange] = useReducer(produce((draft, action) => func.dateRangeReducer(draft, action)), values.ranges[3]); + const getTimeEpoch = (key) => { + return Math.floor(Date.parse(currDateRange.period[key]) / 1000) + } + + const startTimestamp = getTimeEpoch("since") + const endTimestamp = getTimeEpoch("until") + function disambiguateLabel(key, value) { switch (key) { case "location": @@ -144,8 +154,6 @@ function SensitiveDataExposure() { return func.convertToDisambiguateLabelObj(value, apiCollectionMap, 2) case "isRequest": return value[0] ? "In request" : "In response" - case "dateRange": - return value.since.toDateString() + " - " + value.until.toDateString(); default: return value; } @@ -161,14 +169,6 @@ function SensitiveDataExposure() { filterOperators['subType']="OR" let ret = [] let total = 0; - let dateRange = filters['dateRange'] || false; - delete filters['dateRange'] - let startTimestamp = 0; - let endTimestamp = func.timeNow() - if(dateRange){ - startTimestamp = Math.floor(Date.parse(dateRange.since) / 1000); - endTimestamp = Math.floor(Date.parse(dateRange.until) / 1000) - } await api.fetchChanges(sortKey, sortOrder, skip, limit, filters, filterOperators, startTimestamp, endTimestamp, true,isRequest).then((res)=> { res.endpoints.forEach((endpoint, index) => { let temp = {} @@ -196,6 +196,13 @@ const handleRedirect = () => { navigate("/dashboard/observe/data-types") } +const primaryActions = ( + + dispatchCurrDateRange({type: "update", period: dateObj.period, title: dateObj.title, alias: dateObj.alias})}/> + + +) + return ( { } backUrl="/dashboard/observe/sensitive" - primaryAction={} + primaryAction={primaryActions} components = {[ { filters={filters} // promotedBulkActions={promotedBulkActions} hideQueryField={true} - calenderFilter={true} getStatus={func.getTestResultStatus} /> ]} From aa1743e0a5ecac43d4a83fb5eead9c7b006fc79a Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Fri, 12 Apr 2024 17:47:47 +0530 Subject: [PATCH 10/13] fixed issues page filtering --- .../dashboard/pages/issues/IssuesPage/IssuesPage.jsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/issues/IssuesPage/IssuesPage.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/issues/IssuesPage/IssuesPage.jsx index 36b8f2f4bf..714b2cc9e8 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/issues/IssuesPage/IssuesPage.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/issues/IssuesPage/IssuesPage.jsx @@ -253,7 +253,11 @@ function IssuesPage(){ async function fetchData(sortKey, sortOrder, skip, limit, filters, filterOperators, queryValue){ setLoading(true); - + const res = await api.fetchIssues(skip, 1, null, null, null, null, 0) + if(res.totalIssuesCount === 0){ + setShowEmptyScreen(true) + return {value:{} , total:0}; + } let total =0; let ret = [] let filterCollectionsId = filters.apiCollectionId.concat(filters.collectionIds); @@ -279,9 +283,7 @@ function IssuesPage(){ setLoading(false); }) ret = func.sortFunc(ret, sortKey, sortOrder) - if(total === 0){ - setShowEmptyScreen(true) - } + return {value:ret , total:total}; } From 24e70a9a31e7c3ef03ad24168eda2b10696ad352 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Sat, 13 Apr 2024 10:27:46 +0530 Subject: [PATCH 11/13] fixing column index on test runs page --- .../pages/testing/SingleTestRunPage/SingleTestRunPage.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js index 40b5d029dd..a51f6a12c5 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/SingleTestRunPage/SingleTestRunPage.js @@ -68,8 +68,8 @@ let headers = [ const sortOptions = [ { label: 'Severity', value: 'severity asc', directionLabel: 'Highest severity', sortKey: 'total_severity', columnIndex: 2}, { label: 'Severity', value: 'severity desc', directionLabel: 'Lowest severity', sortKey: 'total_severity', columnIndex: 2 }, - { label: 'Run time', value: 'time asc', directionLabel: 'Newest run', sortKey: 'endTimestamp', columnIndex: 5 }, - { label: 'Run time', value: 'time desc', directionLabel: 'Oldest run', sortKey: 'endTimestamp', columnIndex: 5 }, + { label: 'Run time', value: 'time asc', directionLabel: 'Newest run', sortKey: 'endTimestamp', columnIndex: 6 }, + { label: 'Run time', value: 'time desc', directionLabel: 'Oldest run', sortKey: 'endTimestamp', columnIndex: 6 }, ]; const resourceName = { From 3ec5ca90ba1c0190b11262811504ca6822092406 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Mon, 15 Apr 2024 15:39:18 +0530 Subject: [PATCH 12/13] allow sync with db for dependencyAnalyser --- .../src/main/java/com/akto/parsers/HttpCallParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 131c9d492b..b6c59645ff 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -214,7 +214,7 @@ public void syncFunction(List responseParams, boolean syncIm numberOfSyncs++; apiCatalogSync.syncWithDB(syncImmediately, fetchAllSTI); dependencyAnalyser.dbState = apiCatalogSync.dbState; -// dependencyAnalyser.syncWithDb(); + dependencyAnalyser.syncWithDb(); syncTrafficMetricsWithDB(); this.last_synced = Context.now(); this.sync_count = 0; From 0ffd1b9af6f8fb7f563ee21252fe84a51a1680ac Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Mon, 15 Apr 2024 17:54:32 +0530 Subject: [PATCH 13/13] added limit to db calls and iterations --- .../java/com/akto/dto/dependency_flow/DependencyFlow.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/dao/src/main/java/com/akto/dto/dependency_flow/DependencyFlow.java b/libs/dao/src/main/java/com/akto/dto/dependency_flow/DependencyFlow.java index fe01e244eb..c6b890a3c1 100644 --- a/libs/dao/src/main/java/com/akto/dto/dependency_flow/DependencyFlow.java +++ b/libs/dao/src/main/java/com/akto/dto/dependency_flow/DependencyFlow.java @@ -68,7 +68,9 @@ public void run() { Set neverL0 = new HashSet<>(); Set maybeL0 = new HashSet<>(); - while (true) { + int maxIterations = 100; + while (maxIterations > 0) { + maxIterations -= 1; Bson filter = lastId == null ? new BasicDBObject() : Filters.gt(ID, lastId); List dependencyNodeList = DependencyNodeDao.instance.findAll(filter, 0, limit, Sorts.ascending(ID)); findL0Apis(neverL0, maybeL0, dependencyNodeList); @@ -106,7 +108,7 @@ public static ReverseNode buildReverseNode(ApiInfo.ApiInfoKey apiInfoKey) { Filters.eq(DependencyNode.API_COLLECTION_ID_RESP, apiInfoKey.getApiCollectionId()+""), Filters.eq(DependencyNode.URL_RESP, apiInfoKey.getUrl()), Filters.eq(DependencyNode.METHOD_RESP, apiInfoKey.getMethod().name()) - ) + ), 0, 500, Sorts.ascending("_id") ); ReverseNode reverseNode = new ReverseNode(