From 3680fa1e295e39ca03c3cc9335874315ba3d864e Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Tue, 16 Apr 2024 16:45:19 +0530 Subject: [PATCH] added functionality to reduce garbage endpoints --- .../java/com/akto/parsers/HttpCallParser.java | 38 ++++++++++++++++++- .../com/akto/action/AdminSettingsAction.java | 24 ++++++++++-- apps/dashboard/src/main/resources/struts.xml | 13 +++++++ .../dashboard/pages/settings/about/About.jsx | 14 ++++++- .../src/apps/dashboard/pages/settings/api.js | 11 +++++- .../java/com/akto/dto/AccountSettings.java | 13 ++++++- 6 files changed, 105 insertions(+), 8 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index b6c59645ff..4676d95e54 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -31,6 +31,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static com.akto.runtime.RuntimeUtil.matchesDefaultPayload; @@ -53,6 +55,7 @@ public class HttpCallParser { .build(); private static final ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + private static final int MAX_ALLOWED_HTML_CONTENT = 1024 * 1024 ; public static void init() { trafficMetricsExecutor.scheduleAtFixedRate(new Runnable() { @@ -196,7 +199,7 @@ public void syncFunction(List responseParams, boolean syncIm if (accountSettings != null && accountSettings.getDefaultPayloads() != null) { filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads()); } - filteredResponseParams = filterHttpResponseParams(filteredResponseParams); + filteredResponseParams = filterHttpResponseParams(filteredResponseParams, accountSettings); boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap); for (int apiCollectionId: aggregatorMap.keySet()) { @@ -354,7 +357,23 @@ public void incTrafficMetrics(TrafficMetrics.Key key, int value) { trafficMetrics.inc(value); } - public List filterHttpResponseParams(List httpResponseParamsList) { + private boolean isRedundantEndpoint(String url){ + String regex = ".*\\.(js|css|svg|png|json|html|io).*"; + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(url); + return matcher.matches(); + } + + private boolean isInvalidContentType(String contentType){ + boolean res = false; + if(contentType == null || contentType.length() == 0) return res; + + res = contentType.contains("javascript") || contentType.contains("png"); + return res; + } + + public List filterHttpResponseParams(List httpResponseParamsList, AccountSettings accountSettings) { List filteredResponseParams = new ArrayList<>(); int originalSize = httpResponseParamsList.size(); for (HttpResponseParams httpResponseParam: httpResponseParamsList) { @@ -374,6 +393,21 @@ public List filterHttpResponseParams(List contentTypeList = (List) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>()); + String contentType = null; + if(!contentTypeList.isEmpty()){ + contentType = contentTypeList.get(0); + } + if(isInvalidContentType(contentType)){ + continue; + } + } + String hostName = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(), "host"); diff --git a/apps/dashboard/src/main/java/com/akto/action/AdminSettingsAction.java b/apps/dashboard/src/main/java/com/akto/action/AdminSettingsAction.java index 8f7cce951b..abbbeb492d 100644 --- a/apps/dashboard/src/main/java/com/akto/action/AdminSettingsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/AdminSettingsAction.java @@ -3,8 +3,6 @@ import com.akto.dao.*; import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.context.Context; -import com.akto.dto.AccountSettings; -import com.akto.dto.User; import com.akto.dto.type.CollectionReplaceDetails; import com.akto.dto.*; import com.akto.dto.billing.Organization; @@ -50,8 +48,9 @@ public String execute() throws Exception { public Boolean enableTelemetry; private List partnerIpList; + private boolean allowRedundantEndpoints; - public String updateSetupType() { + public String updateSetupType() { AccountSettingsDao.instance.getMCollection().updateOne( AccountSettingsDao.generateFilter(), Updates.set(AccountSettings.SETUP_TYPE, this.setupType), @@ -261,6 +260,21 @@ public String editPartnerIpList(){ } + public String updateUrlSettings() { + try { + AccountSettingsDao.instance.getMCollection().updateOne( + AccountSettingsDao.generateFilter(), + Updates.set(AccountSettings.ALLOW_REDUNDANT_ENDPOINTS, this.allowRedundantEndpoints), + new UpdateOptions().upsert(true) + ); + + return SUCCESS.toUpperCase(); + } catch (Exception e) { + return ERROR.toUpperCase(); + } + + } + public AccountSettings getAccountSettings() { return this.accountSettings; } @@ -345,4 +359,8 @@ public void setPrivateCidrList(List privateCidrList) { public List getPrivateCidrList() { return privateCidrList; } + + public void setAllowRedundantEndpoints(boolean allowRedundantEndpoints) { + this.allowRedundantEndpoints = allowRedundantEndpoints; + } } diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index d67944b067..99b60fdef5 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -1821,6 +1821,19 @@ + + + + + allowRedundantEndpoints + + + 422 + false + ^actionErrors.* + + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/about/About.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/about/About.jsx index d42f3e3036..90360fa738 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/about/About.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/about/About.jsx @@ -36,6 +36,7 @@ function About() { const [enableTelemetry, setEnableTelemetry] = useState(false) const [privateCidrList, setPrivateCidrList] = useState([]) const [partnerIpsList, setPartnerIpsList] = useState([]) + const [allowRedundantUrls,setAllowRedundantUrls] = useState(false) const setupOptions = settingFunctions.getSetupOptions() @@ -57,6 +58,8 @@ function About() { setPrivateCidrList(resp.privateCidrList || []) setPartnerIpsList(resp.partnerIpList || []) + console.log(resp) + setAllowRedundantUrls(resp.allowRedundantEndpoints) } useEffect(()=>{ @@ -112,6 +115,11 @@ function About() { await settingRequests.updateTrafficAlertThresholdSeconds(val); } + const toggleUrlSettings = async(val) => { + setAllowRedundantUrls(val); + await settingRequests.handleRedundantUrls(val); + } + const handleIpsChange = async(ip, isAdded, type) => { if(type === 'cidr'){ let updatedIps = [] @@ -308,7 +316,7 @@ function About() { Details}> {infoComponent} - {isOnPrem && + {isOnPrem ? More settings}>
@@ -326,6 +334,7 @@ function About() { + Traffic alert threshold @@ -345,6 +354,9 @@ function About() {
+ :More settings}> + + } View our terms of service and privacy policy diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js index d0a7e4d656..2dc6162091 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js @@ -412,7 +412,16 @@ const settingRequests = { method: 'post', data: {partnerIpList} }) - } + }, + handleRedundantUrls(allowRedundantEndpoints) { + return request({ + url: '/api/updateUrlSettings', + method: 'post', + data: { + allowRedundantEndpoints + } + }); + }, } export default settingRequests \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dto/AccountSettings.java b/libs/dao/src/main/java/com/akto/dto/AccountSettings.java index 13063249f8..02d085204f 100644 --- a/libs/dao/src/main/java/com/akto/dto/AccountSettings.java +++ b/libs/dao/src/main/java/com/akto/dto/AccountSettings.java @@ -94,7 +94,10 @@ public class AccountSettings { public static final String PARTNER_IP_LIST = "partnerIpList"; private List partnerIpList; - public AccountSettings() { + public static final String ALLOW_REDUNDANT_ENDPOINTS = "allowRedundantEndpoints"; + private boolean allowRedundantEndpoints; + + public AccountSettings() { } public AccountSettings(int id, List privateCidrList, Boolean redactPayload, SetupType setupType) { @@ -360,4 +363,12 @@ public List getPartnerIpList() { public void setPartnerIpList(List partnerIpList) { this.partnerIpList = partnerIpList; } + + public boolean getAllowRedundantEndpoints() { + return allowRedundantEndpoints; + } + + public void setAllowRedundantEndpoints(boolean allowRedundantEndpoints) { + this.allowRedundantEndpoints = allowRedundantEndpoints; + } }