Skip to content

Commit

Permalink
Merge pull request #917 from akto-api-security/develop
Browse files Browse the repository at this point in the history 
Develop to master
  • Loading branch information
@aktoboy
aktoboy authored Feb 29, 2024
2 parents 98994ac + 16deeb3 commit 4e93530
Show file tree
Hide file tree
Showing 19 changed files with 1,094 additions and 184 deletions.
2 changes: 1 addition & 1 deletion apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams
}

boolean cond = HttpResponseParams.validHttpResponseCode(httpResponseParam.getStatusCode());
if (httpResponseParam.getSource().equals(HttpResponseParams.Source.POSTMAN)) {
if (httpResponseParam.getSource().equals(HttpResponseParams.Source.POSTMAN) && httpResponseParam.getStatusCode() <= 0) {
cond = true;
}

Expand Down
21 changes: 10 additions & 11 deletions apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,20 +87,19 @@ public void processResponse(RequestTemplate requestTemplate, HttpResponseParams
if (!responseParams.getIsPending()) {
requestTemplate.processTraffic(responseParams.getTime());
}
if (HttpResponseParams.validHttpResponseCode(statusCode)) {
String reqPayload = requestParams.getPayload();

if (reqPayload == null || reqPayload.isEmpty()) {
reqPayload = "{}";
}
String reqPayload = requestParams.getPayload();

requestTemplate.processHeaders(requestParams.getHeaders(), baseURL.getUrl(), methodStr, -1, userId, requestParams.getApiCollectionId(), responseParams.getOrig(), sensitiveParamInfoBooleanMap);
BasicDBObject payload = RequestTemplate.parseRequestPayload(requestParams, urlWithParams);
if (payload != null) {
deletedInfo.addAll(requestTemplate.process2(payload, baseURL.getUrl(), methodStr, -1, userId, requestParams.getApiCollectionId(), responseParams.getOrig(), sensitiveParamInfoBooleanMap));
}
requestTemplate.recordMessage(responseParams.getOrig());
if (reqPayload == null || reqPayload.isEmpty()) {
reqPayload = "{}";
}

requestTemplate.processHeaders(requestParams.getHeaders(), baseURL.getUrl(), methodStr, -1, userId, requestParams.getApiCollectionId(), responseParams.getOrig(), sensitiveParamInfoBooleanMap);
BasicDBObject requestPayload = RequestTemplate.parseRequestPayload(requestParams, urlWithParams);
if (requestPayload != null) {
deletedInfo.addAll(requestTemplate.process2(requestPayload, baseURL.getUrl(), methodStr, -1, userId, requestParams.getApiCollectionId(), responseParams.getOrig(), sensitiveParamInfoBooleanMap));
}
requestTemplate.recordMessage(responseParams.getOrig());

Map<Integer, RequestTemplate> responseTemplates = requestTemplate.getResponseTemplates();

Expand Down
2 changes: 1 addition & 1 deletion apps/dashboard/src/main/java/com/akto/action/HarAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ public String execute() throws IOException {
HAR har = new HAR();
loggerMaker.infoAndAddToDb("Har file upload processing for collectionId:" + apiCollectionId, LoggerMaker.LogDb.DASHBOARD);
String zippedString = GzipUtils.zipString(harString);
com.akto.dto.files.File file = new com.akto.dto.files.File(HttpResponseParams.Source.HAR,zippedString);
com.akto.dto.files.File file = new com.akto.dto.files.File(HttpResponseParams.Source.HAR.toString(),zippedString);
FilesDao.instance.insertOne(file);
List<String> messages = har.getMessages(harString, apiCollectionId, Context.accountId.get());
harErrors = har.getErrors();
Expand Down
429 changes: 348 additions & 81 deletions apps/dashboard/src/main/java/com/akto/action/PostmanAction.java
View file

Large diffs are not rendered by default.

16 changes: 16 additions & 0 deletions apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import com.akto.dao.testing.*;
import com.akto.dao.testing_run_findings.TestingRunIssuesDao;
import com.akto.dao.traffic_metrics.TrafficMetricsDao;
import com.akto.dao.upload.FileUploadLogsDao;
import com.akto.dao.upload.FileUploadsDao;
import com.akto.dao.usage.UsageMetricInfoDao;
import com.akto.dao.usage.UsageMetricsDao;
import com.akto.dto.*;
Expand All @@ -44,6 +46,7 @@
import com.akto.dto.traffic.Key;
import com.akto.dto.traffic.SampleData;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.dto.upload.FileUpload;
import com.akto.dto.usage.MetricTypes;
import com.akto.dto.usage.UsageMetric;
import com.akto.dto.usage.UsageMetricInfo;
Expand Down Expand Up @@ -2243,6 +2246,19 @@ public void run() {
}
}, 0, 1, UsageUtils.USAGE_CRON_PERIOD);
}

public static void deleteFileUploads(int accountId){
Context.accountId.set(accountId);
List<FileUpload> markedForDeletion = FileUploadsDao.instance.findAll(eq("markedForDeletion", true));
loggerMaker.infoAndAddToDb(String.format("Deleting %d file uploads", markedForDeletion.size()), LogDb.DASHBOARD);
for (FileUpload fileUpload : markedForDeletion) {
loggerMaker.infoAndAddToDb(String.format("Deleting file upload logs for uploadId: %s", fileUpload.getId()), LogDb.DASHBOARD);
FileUploadLogsDao.instance.deleteAll(eq("uploadId", fileUpload.getId().toString()));
loggerMaker.infoAndAddToDb(String.format("Deleting file upload: %s", fileUpload.getId()), LogDb.DASHBOARD);
FileUploadsDao.instance.deleteAll(eq("_id", fileUpload.getId()));
loggerMaker.infoAndAddToDb(String.format("Deleted file upload: %s", fileUpload.getId()), LogDb.DASHBOARD);
}
}

}

176 changes: 98 additions & 78 deletions apps/dashboard/src/main/java/com/akto/utils/Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.akto.dto.third_party_access.PostmanCredential;
import com.akto.dto.third_party_access.ThirdPartyAccess;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.dto.upload.FileUploadError;
import com.akto.listener.KafkaListener;
import com.akto.listener.RuntimeListener;
import com.akto.log.LoggerMaker;
Expand All @@ -28,6 +29,7 @@
import com.mongodb.BasicDBObject;
import com.mongodb.client.model.Filters;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;

import java.net.MalformedURLException;
import java.net.URISyntaxException;
Expand All @@ -51,47 +53,47 @@ public static Map<String, String> getAuthMap(JsonNode auth, Map<String, String>
return result;
}

try {
String authType = auth.get("type").asText().toLowerCase();

switch (authType) {
case "bearer":
ArrayNode authParams = (ArrayNode) auth.get("bearer");
for (JsonNode authHeader : authParams) {
String tokenKey = authHeader.get("key").asText();
if (tokenKey.equals("token")) {
String tokenValue = authHeader.get("value").asText();
String replacedTokenValue = replaceVariables(tokenValue, variableMap);
String authType = auth.get("type").asText().toLowerCase();

result.put("Authorization", "Bearer " + replacedTokenValue);
switch (authType) {
case "bearer":
ArrayNode authParams = (ArrayNode) auth.get("bearer");
for (JsonNode authHeader : authParams) {
String tokenKey = authHeader.get("key").asText();
if (tokenKey.equals("token")) {
String tokenValue = authHeader.get("value").asText();
String replacedTokenValue = replaceVariables(tokenValue, variableMap);

result.put("Authorization", "Bearer " + replacedTokenValue);

}
}
break;
case "apikey":
ArrayNode apikeyParams = (ArrayNode) auth.get("apikey");
String authKeyName = "", authValueName = "";
for (JsonNode apikeyHeader : apikeyParams) {
String key = apikeyHeader.get("key").asText();
String value = apikeyHeader.get("value").asText();

switch (key) {
case "key":
authKeyName = replaceVariables(value, variableMap);
break;
case "value":
authValueName = replaceVariables(value, variableMap);
break;

case "in":
if (!value.equals("header")) {
throw new IllegalArgumentException("Only header supported in apikey");
}
break;
default:
break;
}
}
break;
case "apikey":
ArrayNode apikeyParams = (ArrayNode) auth.get("apikey");
String authKeyName = "", authValueName = "";
for (JsonNode apikeyHeader : apikeyParams) {
String key = apikeyHeader.get("key").asText();
String value = apikeyHeader.get("value").asText();

switch (key) {
case "key":
authKeyName = replaceVariables(value, variableMap);
break;
case "value":
authValueName = replaceVariables(value, variableMap);
break;

case "in":
if (!value.equals("header")) {
throw new IllegalArgumentException("Only header supported in apikey");
}
break;
default:
break;
}
}

if (authKeyName.isEmpty() || authValueName.isEmpty()) {
throw new IllegalArgumentException(
Expand All @@ -100,45 +102,43 @@ public static Map<String, String> getAuthMap(JsonNode auth, Map<String, String>
result.put(authKeyName, authValueName);
}
break;
case "basic":
ArrayNode basicParams = (ArrayNode) auth.get("basic");
String basicUsername = "", basicPassword = "";
for (JsonNode basicKeyHeader : basicParams) {
String key = basicKeyHeader.get("key").asText();
String value = basicKeyHeader.get("value").asText();
switch (key) {
case "username":
basicUsername = replaceVariables(value, variableMap);
break;
case "password":
basicPassword = replaceVariables(value, variableMap);
break;
default:
break;
}
case "basic":
ArrayNode basicParams = (ArrayNode) auth.get("basic");
String basicUsername = "", basicPassword = "";
for (JsonNode basicKeyHeader : basicParams) {
String key = basicKeyHeader.get("key").asText();
String value = basicKeyHeader.get("value").asText();
switch (key) {
case "username":
basicUsername = replaceVariables(value, variableMap);
break;
case "password":
basicPassword = replaceVariables(value, variableMap);
break;
default:
break;
}
}

if (basicUsername.isEmpty() || basicPassword.isEmpty()) {
throw new IllegalArgumentException(
"One of username/password is empty: username=" + basicUsername + " password="
+ basicPassword);
} else {
/*
* Base64 implementation ref: https://www.ietf.org/rfc/rfc2617.txt
*/
String basicCredentials = basicUsername + ":" + basicPassword;
String basicEncoded = Base64.getEncoder().encodeToString(basicCredentials.getBytes());

String basicHeader = "Basic " + basicEncoded;
result.put("Authorization", basicHeader);
}
break;
if (basicUsername.isEmpty() || basicPassword.isEmpty()) {
throw new IllegalArgumentException(
"One of username/password is empty: username=" + basicUsername + " password="
+ basicPassword);
} else {
/*
* Base64 implementation ref: https://www.ietf.org/rfc/rfc2617.txt
*/
String basicCredentials = basicUsername + ":" + basicPassword;
String basicEncoded = Base64.getEncoder().encodeToString(basicCredentials.getBytes());

String basicHeader = "Basic " + basicEncoded;
result.put("Authorization", basicHeader);
}
break;
default:
throw new IllegalArgumentException("Unsupported auth type: " + authType );
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb("Unable to parse auth from postman file: " + e.getMessage(), LogDb.DASHBOARD);
}


return result;
}
Expand Down Expand Up @@ -190,23 +190,33 @@ public static String replaceVariables(String payload, Map<String, String> variab
return sb.toString();
}

public static Map<String, String> convertApiInAktoFormat(JsonNode apiInfo, Map<String, String> variables, String accountId, boolean allowReplay, Map<String, String> authMap) {
public static Pair<Map<String, String>, List<FileUploadError>> convertApiInAktoFormat(JsonNode apiInfo, Map<String, String> variables, String accountId, boolean allowReplay, Map<String, String> authMap) {
Pair<Map<String, String>, List<String>> resp;
List<FileUploadError> errors = new ArrayList<>();
try {
JsonNode request = apiInfo.get("request");

String apiName = apiInfo.get("name").asText();

JsonNode authApiNode = request.get("auth");
if (authApiNode != null) {
authMap = getAuthMap(authApiNode, variables);
try {
authMap = getAuthMap(authApiNode, variables);
}catch (Exception e) {
errors.add(new FileUploadError("Error while getting auth map for " + apiName + " : " + e.toString(), FileUploadError.ErrorType.WARNING));
loggerMaker.errorAndAddToDb(e, String.format("Error while getting auth map for %s : %s", apiName, e.toString()), LogDb.DASHBOARD);
}
}

Map<String, String> result = new HashMap<>();
result.put("akto_account_id", accountId);
result.put("path", getPath(request, variables));

JsonNode methodObj = request.get("method");
if (methodObj == null) throw new Exception("No method field exists");
if (methodObj == null){
errors.add(new FileUploadError("No method field exists", FileUploadError.ErrorType.ERROR));
return Pair.of(null, errors);
}
result.put("method", methodObj.asText());

ArrayNode requestHeadersNode = (ArrayNode) request.get("header");
Expand Down Expand Up @@ -241,12 +251,17 @@ public static Map<String, String> convertApiInAktoFormat(JsonNode apiInfo, Map<S
responsePayload = res.getBody();
statusCode = res.getStatusCode()+"";
status = "";
if(res.getStatusCode() < 200 || res.getStatusCode() >=400){
throw new Exception("Found non 2XX response on replaying the API");
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e,"Error while making request for " + originalHttpRequest.getFullUrlWithParams() + " : " + e.toString(), LogDb.DASHBOARD);
return null;
errors.add(new FileUploadError("Error while replaying request for " + originalHttpRequest.getFullUrlWithParams() + " : " + e.toString(), FileUploadError.ErrorType.ERROR));
return Pair.of(null, errors);
}
} else {
return null;
errors.add(new FileUploadError("No response field exists", FileUploadError.ErrorType.WARNING));
return Pair.of(null, errors);
}
} else {
JsonNode respHeaders = response.get("header");
Expand Down Expand Up @@ -282,11 +297,11 @@ public static Map<String, String> convertApiInAktoFormat(JsonNode apiInfo, Map<S
result.put("time", Context.now()+"");
result.put("type", "http");
result.put("source", "POSTMAN");

return result;
return Pair.of(result, errors);
} catch (Exception e){
loggerMaker.errorAndAddToDb(e, String.format("Failed to convert postman obj to Akto format : %s", e.toString()), LogDb.DASHBOARD);
return null;
errors.add(new FileUploadError("Failed to convert postman obj to Akto", FileUploadError.ErrorType.ERROR));
return Pair.of(null, errors);
}
}

Expand Down Expand Up @@ -352,6 +367,10 @@ private static String getContentType(JsonNode request, JsonNode response, Map<St
return response.get("_postman_previewlanguage").asText();
}

public static String getPath(JsonNode request) throws Exception {
return getPath(request, new HashMap<>());
}

public static String getPath(JsonNode request, Map<String, String> variables) throws Exception {
JsonNode urlObj = request.get("url");
if (urlObj == null) throw new Exception("URL field doesn't exists");
Expand Down Expand Up @@ -401,6 +420,7 @@ private static Map<String, String> getHeaders(ArrayNode headers, Map<String, Str
return result;
}

//TODO handle item as a json object
public static void fetchApisRecursively(ArrayNode items, ArrayList<JsonNode> jsonNodes) {
if(items == null || items.size() == 0){
return;
Expand Down
7 changes: 7 additions & 0 deletions apps/dashboard/src/main/java/com/akto/utils/crons/Crons.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;

import com.akto.dao.context.Context;
import com.akto.listener.InitializerListener;
import com.akto.log.LoggerMaker;
import org.bson.types.ObjectId;

import com.akto.dao.testing.DeleteTestRunsDao;
Expand All @@ -17,6 +20,7 @@
public class Crons {

ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
private static LoggerMaker logger = new LoggerMaker(Crons.class, LoggerMaker.LogDb.DASHBOARD);

public void deleteTestRunsScheduler(){
scheduler.scheduleAtFixedRate(new Runnable() {
Expand All @@ -36,6 +40,9 @@ public void accept(Account t) {
}
}
}
logger.infoAndAddToDb("Starting to delete pending test runs");
InitializerListener.deleteFileUploads(Context.accountId.get());
logger.infoAndAddToDb("Finished deleting pending test runs");
} catch (Exception e) {
e.printStackTrace();
}
Expand Down
Loading

0 comments on commit 4e93530

Please sign in to comment.