added functionality to reduce garbage endpoints

akto-api-security · Apr 16, 2024 · 3680fa1 · 3680fa1
1 parent bdc7412
commit 3680fa1
Show file tree

Hide file tree

Showing 6 changed files with 105 additions and 8 deletions.
diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
@@ -31,6 +31,8 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static com.akto.runtime.RuntimeUtil.matchesDefaultPayload;
 
@@ -53,6 +55,7 @@ public class HttpCallParser {
             .build();
 
     private static final ConcurrentLinkedQueue<BasicDBObject> queue = new ConcurrentLinkedQueue<>();
+    private static final int MAX_ALLOWED_HTML_CONTENT = 1024 * 1024 ;
 
     public static void init() {
         trafficMetricsExecutor.scheduleAtFixedRate(new Runnable() {
@@ -196,7 +199,7 @@ public void syncFunction(List<HttpResponseParams> responseParams, boolean syncIm
         if (accountSettings != null && accountSettings.getDefaultPayloads() != null) {
             filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads());
         }
-        filteredResponseParams = filterHttpResponseParams(filteredResponseParams);
+        filteredResponseParams = filterHttpResponseParams(filteredResponseParams, accountSettings);
         boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap);
 
         for (int apiCollectionId: aggregatorMap.keySet()) {
@@ -354,7 +357,23 @@ public void incTrafficMetrics(TrafficMetrics.Key key, int value)  {
         trafficMetrics.inc(value);
     }
 
-    public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams> httpResponseParamsList) {
+    private boolean isRedundantEndpoint(String url){
+        String regex = ".*\\.(js|css|svg|png|json|html|io).*";
+
+        Pattern pattern = Pattern.compile(regex);
+        Matcher matcher = pattern.matcher(url);
+        return matcher.matches();
+    }
+
+    private boolean isInvalidContentType(String contentType){
+        boolean res = false;
+        if(contentType == null || contentType.length() == 0) return res;
+
+        res = contentType.contains("javascript") || contentType.contains("png");
+        return res;
+    }
+
+    public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams> httpResponseParamsList, AccountSettings accountSettings) {
         List<HttpResponseParams> filteredResponseParams = new ArrayList<>();
         int originalSize = httpResponseParamsList.size();
         for (HttpResponseParams httpResponseParam: httpResponseParamsList) {
@@ -374,6 +393,21 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams
             String ignoreAktoFlag = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(),Constants.AKTO_IGNORE_FLAG);
             if (ignoreAktoFlag != null) continue;
 
+            // check for garbage points here
+            if(!accountSettings.getAllowRedundantEndpoints()){
+                if(isRedundantEndpoint(httpResponseParam.getRequestParams().getURL())){
+                    continue;
+                }
+                List<String> contentTypeList = (List<String>) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>());
+                String contentType = null;
+                if(!contentTypeList.isEmpty()){
+                    contentType = contentTypeList.get(0);
+                }
+                if(isInvalidContentType(contentType)){
+                    continue;
+                }
+            }
+
             String hostName = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(), "host");
 
 

diff --git a/apps/dashboard/src/main/java/com/akto/action/AdminSettingsAction.java b/apps/dashboard/src/main/java/com/akto/action/AdminSettingsAction.java
@@ -3,8 +3,6 @@
 import com.akto.dao.*;
 import com.akto.dao.billing.OrganizationsDao;
 import com.akto.dao.context.Context;
-import com.akto.dto.AccountSettings;
-import com.akto.dto.User;
 import com.akto.dto.type.CollectionReplaceDetails;
 import com.akto.dto.*;
 import com.akto.dto.billing.Organization;
@@ -50,8 +48,9 @@ public String execute() throws Exception {
     public Boolean enableTelemetry;
 
 	private List<String> partnerIpList;
+    private boolean allowRedundantEndpoints;
 
-	public String updateSetupType() {
+    public String updateSetupType() {
         AccountSettingsDao.instance.getMCollection().updateOne(
                 AccountSettingsDao.generateFilter(),
                 Updates.set(AccountSettings.SETUP_TYPE, this.setupType),
@@ -261,6 +260,21 @@ public String editPartnerIpList(){
 
     }
 
+    public String updateUrlSettings() {
+        try {
+            AccountSettingsDao.instance.getMCollection().updateOne(
+                AccountSettingsDao.generateFilter(),
+                Updates.set(AccountSettings.ALLOW_REDUNDANT_ENDPOINTS, this.allowRedundantEndpoints),
+                new UpdateOptions().upsert(true)
+            );
+
+            return SUCCESS.toUpperCase();
+        } catch (Exception e) {
+            return ERROR.toUpperCase();
+        }
+
+    }
+
     public AccountSettings getAccountSettings() {
         return this.accountSettings;
     }
@@ -345,4 +359,8 @@ public void setPrivateCidrList(List<String> privateCidrList) {
     public List<String> getPrivateCidrList() {
 		return privateCidrList;
 	}
+
+    public void setAllowRedundantEndpoints(boolean allowRedundantEndpoints) {
+        this.allowRedundantEndpoints = allowRedundantEndpoints;
+    }
 }
diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml
@@ -1821,6 +1821,19 @@
             </result>
         </action>
 
+        <action name="api/updateUrlSettings" class="com.akto.action.AdminSettingsAction" method="updateUrlSettings">
+            <interceptor-ref name="json"/>
+            <interceptor-ref name="defaultStack" />
+            <result name="SUCCESS" type="json">
+                <param name="root">allowRedundantEndpoints</param>
+            </result>
+            <result name="ERROR" type="json">
+                <param name="statusCode">422</param>
+                <param name="ignoreHierarchy">false</param>
+                <param name="includeProperties">^actionErrors.*</param>
+            </result>
+        </action>
+
         <action name="api/fetchParamsStatus" class="com.akto.action.ParamStateAction" method="fetchParamsStatus">
             <interceptor-ref name="json"/>
             <interceptor-ref name="defaultStack" />

diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/about/About.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/about/About.jsx
@@ -36,6 +36,7 @@ function About() {
     const [enableTelemetry, setEnableTelemetry] = useState(false)
     const [privateCidrList, setPrivateCidrList] = useState([])
     const [partnerIpsList, setPartnerIpsList] = useState([])
+    const [allowRedundantUrls,setAllowRedundantUrls] = useState(false)
 
     const setupOptions = settingFunctions.getSetupOptions()
 
@@ -57,6 +58,8 @@ function About() {
 
         setPrivateCidrList(resp.privateCidrList || [])
         setPartnerIpsList(resp.partnerIpList || [])
+        console.log(resp)
+        setAllowRedundantUrls(resp.allowRedundantEndpoints)
     }
 
     useEffect(()=>{
@@ -112,6 +115,11 @@ function About() {
         await settingRequests.updateTrafficAlertThresholdSeconds(val);
     }
 
+    const toggleUrlSettings = async(val) => {
+        setAllowRedundantUrls(val);
+        await settingRequests.handleRedundantUrls(val);
+    }
+
     const handleIpsChange = async(ip, isAdded, type) => {
         if(type === 'cidr'){
             let updatedIps = []
@@ -308,7 +316,7 @@ function About() {
               <LegacyCard.Section title={<Text variant="headingMd">Details</Text>}>
                   {infoComponent}
               </LegacyCard.Section>
-              {isOnPrem &&
+              {isOnPrem ?
                   <LegacyCard.Section title={<Text variant="headingMd">More settings</Text>}>
                       <div style={{ display: 'flex' }}>
                           <div style={{ flex: "1" }}>
@@ -326,6 +334,7 @@ function About() {
                                   <ToggleComponent text={"Redact sample data"} initial={redactPayload} onToggle={handleRedactPayload} />
                                   <ToggleComponent text={"Activate regex matching in merging"} initial={newMerging} onToggle={handleNewMerging} />
                                   <ToggleComponent text={"Enable telemetry"} initial={enableTelemetry} onToggle={toggleTelemetry} />
+                                  <ToggleComponent text={"Allow redundant urls"} initial={allowRedundantUrls} onToggle={toggleUrlSettings} />
                                   <VerticalStack gap={1}>
                                       <Text color="subdued">Traffic alert threshold</Text>
                                       <Box width='120px'>
@@ -345,6 +354,9 @@ function About() {
                           </div>
                       </div>
                   </LegacyCard.Section>
+                  :<LegacyCard.Section title={<Text variant="headingMd">More settings</Text>}>
+                    <ToggleComponent text={"Allow redundant urls"} initial={allowRedundantUrls} onToggle={toggleUrlSettings} />
+                  </LegacyCard.Section>
               }
             <LegacyCard.Section subdued>
                 View our <a href='https://www.akto.io/terms-and-policies' target="_blank">terms of service</a> and <a href='https://www.akto.io/terms/privacy' target="_blank" >privacy policy  </a>

diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js
@@ -412,7 +412,16 @@ const settingRequests = {
             method: 'post',
             data: {partnerIpList}
         })
-    }
+    },
+    handleRedundantUrls(allowRedundantEndpoints) {
+        return request({
+            url: '/api/updateUrlSettings',
+            method: 'post',
+            data: {
+                allowRedundantEndpoints
+            }
+        });
+    },
 }
 
 export default settingRequests
diff --git a/libs/dao/src/main/java/com/akto/dto/AccountSettings.java b/libs/dao/src/main/java/com/akto/dto/AccountSettings.java
@@ -94,7 +94,10 @@ public class AccountSettings {
     public static final String PARTNER_IP_LIST = "partnerIpList";
     private List<String> partnerIpList;
 
-	public AccountSettings() {
+    public static final String ALLOW_REDUNDANT_ENDPOINTS = "allowRedundantEndpoints";
+    private boolean allowRedundantEndpoints;
+
+    public AccountSettings() {
     }
 
     public AccountSettings(int id, List<String> privateCidrList, Boolean redactPayload, SetupType setupType) {
@@ -360,4 +363,12 @@ public List<String> getPartnerIpList() {
 	public void setPartnerIpList(List<String> partnerIpList) {
 		this.partnerIpList = partnerIpList;
 	}
+
+    public boolean getAllowRedundantEndpoints() {
+        return allowRedundantEndpoints;
+    }
+
+    public void setAllowRedundantEndpoints(boolean allowRedundantEndpoints) {
+        this.allowRedundantEndpoints = allowRedundantEndpoints;
+    }
 }