Skip to content

Commit

Permalink
Merge branch 'develop' into feature/expired_page
Browse files Browse the repository at this point in the history
  • Loading branch information
@notshivansh
notshivansh committed Feb 10, 2024
2 parents 364f543 + 5444285 commit 4781f8b
Show file tree
Hide file tree
Showing 55 changed files with 892 additions and 326 deletions.
4 changes: 4 additions & 0 deletions apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,10 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams
}

boolean cond = HttpResponseParams.validHttpResponseCode(httpResponseParam.getStatusCode());
if (httpResponseParam.getSource().equals(HttpResponseParams.Source.POSTMAN)) {
cond = true;
}

if (!cond) continue;

String ignoreAktoFlag = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(), AccountSettings.AKTO_IGNORE_FLAG);
Expand Down
212 changes: 151 additions & 61 deletions apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion apps/api-runtime/src/main/java/com/akto/runtime/Main.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ public static void initializeRuntimeHelper() {
for(SingleTypeInfo singleTypeInfo: SingleTypeInfoDao.instance.fetchAll()) {
urls.add(singleTypeInfo.getUrl());
}
ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "Default", Context.now(), urls, null, 0));
ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "Default", Context.now(), urls, null, 0, false, true));
}
}

Expand Down
16 changes: 13 additions & 3 deletions apps/api-runtime/src/main/java/com/akto/runtime/URLAggregator.java
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,37 @@
package com.akto.runtime;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Pattern;

import com.akto.dto.type.KeyTypes;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.dto.type.URLStatic;
import com.akto.dto.type.URLMethods.Method;
import com.akto.dao.context.Context;
import com.akto.dto.CustomDataType;
import com.akto.dto.HttpResponseParams;
import com.mongodb.BasicDBObject;
import com.akto.dto.type.URLTemplate;
import com.akto.dto.type.SingleTypeInfo.SubType;

import org.apache.commons.lang3.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static com.akto.dto.type.KeyTypes.patternToSubType;
import static com.akto.runtime.APICatalogSync.isAlphanumericString;
import static com.akto.runtime.APICatalogSync.tokenize;

public class URLAggregator {

private static final Logger logger = LoggerFactory.getLogger(URLAggregator.class);

ConcurrentMap<URLStatic, Set<HttpResponseParams>> urls;

public static URLStatic getBaseURL(String url, String method) {

if (url == null) {
return null;
}
Expand Down
2 changes: 1 addition & 1 deletion apps/api-runtime/src/main/java/com/akto/runtime/merge/MergeOnHostOnly.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ public void mergeHostUtil(String host, List<Integer> apiCollectionIds) {
old.setId(newApiCollectionId);

try {
ApiCollectionsDao.instance.insertOne(new ApiCollection(newApiCollectionId, null, old.getStartTs(), new HashSet<>(), host, 0));
ApiCollectionsDao.instance.insertOne(new ApiCollection(newApiCollectionId, null, old.getStartTs(), new HashSet<>(), host, 0, false, true));
} catch (Exception e) {
return;
}
Expand Down
74 changes: 56 additions & 18 deletions apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
package com.akto.utils;

import com.akto.dto.HttpRequestParams;
import com.akto.dto.HttpResponseParams;
import com.akto.dto.OriginalHttpRequest;
import com.akto.dto.OriginalHttpResponse;
import com.akto.dto.*;
import com.akto.dto.type.KeyTypes;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.parsers.HttpCallParser;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
Expand All @@ -15,7 +14,6 @@
import com.fasterxml.jackson.databind.node.TextNode;
import com.mongodb.BasicDBObject;

import java.io.IOException;
import java.util.*;

import static com.akto.dto.RawApi.convertHeaders;
Expand All @@ -26,25 +24,47 @@ public class RedactSampleData {

public static final String redactValue = "****";

public static String redact(String sample) throws Exception {
public static String redactIfRequired(String sample, boolean accountLevelRedact, boolean apiCollectionLevelRedact) throws Exception {
HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample);
return redact(httpResponseParams);
HttpResponseParams.Source source = httpResponseParams.getSource();
if(source.equals(HttpResponseParams.Source.HAR) || source.equals(HttpResponseParams.Source.PCAP)) return sample;
return redact(httpResponseParams, accountLevelRedact || apiCollectionLevelRedact);
}

public static String redactDataTypes(String sample) throws Exception{
return redact(HttpCallParser.parseKafkaMessage(sample), false);
}

private static void handleHeaders(Map<String, List<String>> responseHeaders, boolean redactAll) {
if(redactAll){
responseHeaders.replaceAll((n, v) -> Collections.singletonList(redactValue));
return;
}
Set<Map.Entry<String, List<String>>> entries = responseHeaders.entrySet();
for(Map.Entry<String, List<String>> entry : entries){
String key = entry.getKey();
List<String> values = entry.getValue();
SingleTypeInfo.SubType subType = KeyTypes.findSubType(values.get(0), key, null);
if(SingleTypeInfo.isRedacted(subType.getName())){
responseHeaders.put(key, Collections.singletonList(redactValue));
}
}
}

// never use this function directly. This alters the httpResponseParams
public static String redact(HttpResponseParams httpResponseParams) throws Exception {
public static String redact(HttpResponseParams httpResponseParams, final boolean redactAll) throws Exception {
// response headers
Map<String, List<String>> responseHeaders = httpResponseParams.getHeaders();
if (responseHeaders == null) responseHeaders = new HashMap<>();
responseHeaders.replaceAll((n, v) -> Collections.singletonList(redactValue));
handleHeaders(responseHeaders, redactAll);

// response payload
String responsePayload = httpResponseParams.getPayload();
if (responsePayload == null) responsePayload = "{}";
try {
JsonParser jp = factory.createParser(responsePayload);
JsonNode node = mapper.readTree(jp);
change(node, redactValue);
change(null, node, redactValue, redactAll);
if (node != null) {
responsePayload = node.toString();
} else {
Expand All @@ -59,15 +79,15 @@ public static String redact(HttpResponseParams httpResponseParams) throws Except
// request headers
Map<String, List<String>> requestHeaders = httpResponseParams.requestParams.getHeaders();
if (requestHeaders == null) requestHeaders = new HashMap<>();
requestHeaders.replaceAll((n, v) -> Collections.singletonList(redactValue));
handleHeaders(requestHeaders, redactAll);

// request payload
String requestPayload = httpResponseParams.requestParams.getPayload();
if (requestPayload == null) requestPayload = "{}";
try {
JsonParser jp = factory.createParser(requestPayload);
JsonNode node = mapper.readTree(jp);
change(node, redactValue);
change(null, node, redactValue, redactAll);
if (node != null) {
requestPayload= node.toString();
} else {
Expand All @@ -80,23 +100,32 @@ public static String redact(HttpResponseParams httpResponseParams) throws Except
httpResponseParams.requestParams.setPayload(requestPayload);

// ip
httpResponseParams.setSourceIP(redactValue);
if(redactAll) {
httpResponseParams.setSourceIP(redactValue);
}

return convertHttpRespToOriginalString(httpResponseParams);

}

public static void change(JsonNode parent, String newValue) {
public static void change(String parentName, JsonNode parent, String newValue, boolean redactAll) {
if (parent == null) return;

if (parent.isArray()) {
ArrayNode arrayNode = (ArrayNode) parent;
for(int i = 0; i < arrayNode.size(); i++) {
JsonNode arrayElement = arrayNode.get(i);
if (arrayElement.isValueNode()) {
arrayNode.set(i, new TextNode(newValue));
if(redactAll){
arrayNode.set(i, new TextNode(redactValue));
} else{
SingleTypeInfo.SubType subType = KeyTypes.findSubType(arrayElement.asText(), parentName, null);
if(SingleTypeInfo.isRedacted(subType.getName())){
arrayNode.set(i, new TextNode(newValue));
}
}
} else {
change(arrayElement, newValue);
change(parentName, arrayElement, newValue, redactAll);
}
}
} else {
Expand All @@ -105,9 +134,18 @@ public static void change(JsonNode parent, String newValue) {
String f = fieldNames.next();
JsonNode fieldValue = parent.get(f);
if (fieldValue.isValueNode()) {
((ObjectNode) parent).put(f, newValue);
if(redactAll){
((ObjectNode) parent).put(f, newValue);
}
else {
SingleTypeInfo.SubType subType = KeyTypes.findSubType(fieldValue.asText(), f, null);
if (SingleTypeInfo.isRedacted(subType.getName())) {
((ObjectNode) parent).put(f, newValue);
}
}

} else {
change(fieldValue, newValue);
change(f, fieldValue, newValue, redactAll);
}
}
}
Expand Down
30 changes: 15 additions & 15 deletions apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,14 @@ public void changeAccountId() {

public void testInitializer(){
Map<String, AktoDataType> aktoDataTypeMap = new HashMap<>();
aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID);
aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true));
aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true));
aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true));
aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true));
aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true));
aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true));
aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true));
aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()), false, true)); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID);
if (info == null) {
info = new AccountDataTypesInfo();
}
Expand Down Expand Up @@ -110,7 +110,7 @@ public void testParameterizedURL() {
assertEquals(3, respTemplate.getParameters().size());
}

@Test
// @Test
public void testImmediateSync() {
testInitializer();
String url = "immediate/";
Expand Down Expand Up @@ -208,7 +208,7 @@ public void testInvalidMergeParameterizedURL() {
APICatalogSync sync = new APICatalogSync("access-token", 1, true);

for (int i = 1; i <= 30; i ++ ) {
aggr.addURL(TestDump2.createSampleParams("user"+i, "payment/id"+i));
aggr.addURL(TestDump2.createSampleParams("user"+i, "/payment/id"+i));
}
sync.computeDelta(aggr, true, 123);
sync.syncWithDB(false, true);
Expand All @@ -217,7 +217,7 @@ public void testInvalidMergeParameterizedURL() {
assertEquals(30, sync.getDbState(123).getStrictURLToMethods().size());
assertEquals(0, sync.getDbState(123).getTemplateURLToMethods().size());

HttpResponseParams resp2 = TestDump2.createSampleParams("user1", "payment/history");
HttpResponseParams resp2 = TestDump2.createSampleParams("user1", "/payment/history");
ArrayList<String> newHeader = new ArrayList<>();
newHeader.add("hnew");
resp2.getHeaders().put("new header", newHeader);
Expand Down Expand Up @@ -293,7 +293,7 @@ public void testFilterHttpResponseParamsWithoutHost() {
int vxlanId1 = 1;
String domain1 = "domain1.com";

ApiCollectionsDao.instance.insertOne(new ApiCollection(vxlanId1, groupName1, 0, new HashSet<>(), null, 0));
ApiCollectionsDao.instance.insertOne(new ApiCollection(vxlanId1, groupName1, 0, new HashSet<>(), null, 0, false, true));

HttpResponseParams h1 = new HttpResponseParams();
h1.requestParams = new HttpRequestParams();
Expand Down Expand Up @@ -354,7 +354,7 @@ public void testFilterResponseParamsWithHost() {
int vxlanId1 = 1;
String domain1 = "domain1.com";

ApiCollectionsDao.instance.insertOne(new ApiCollection(vxlanId1, groupName1, 0, new HashSet<>(), null, 0));
ApiCollectionsDao.instance.insertOne(new ApiCollection(vxlanId1, groupName1, 0, new HashSet<>(), null, 0, false, true));

HttpResponseParams h1 = new HttpResponseParams();
h1.requestParams = new HttpRequestParams();
Expand Down Expand Up @@ -443,7 +443,7 @@ public void testFilterResponseParamsWithHost() {
// before processing inserting apiCollection with same id but different vxlanId and host
int dupId = domain4.hashCode();
ApiCollectionsDao.instance.insertOne(
new ApiCollection(dupId,"something", 0, new HashSet<>(), "hostRandom", 1234)
new ApiCollection(dupId,"something", 0, new HashSet<>(), "hostRandom", 1234, false, true)
);
httpCallParser.getHostNameToIdMap().put("hostRandom 1234", dupId);

Expand All @@ -461,7 +461,7 @@ public void testFilterResponseParamsWithHost() {
@Test
public void testCollisionHostNameCollection() {
ApiCollectionsDao.instance.getMCollection().drop();
ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "domain", 0, new HashSet<>(), null, 0));
ApiCollectionsDao.instance.insertOne(new ApiCollection(0, "domain", 0, new HashSet<>(), null, 0, false, true));
HttpResponseParams h1 = new HttpResponseParams();
h1.setSource(Source.HAR);
h1.requestParams = new HttpRequestParams();
Expand Down
Loading

0 comments on commit 4781f8b

Please sign in to comment.