Skip to content

Commit

Permalink
Merge pull request #1334 from akto-api-security/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@notshivansh
notshivansh authored Aug 7, 2024
2 parents c28d6a1 + ff74693 commit 7a64f51
Show file tree
Hide file tree
Showing 6 changed files with 86 additions and 28 deletions.
8 changes: 6 additions & 2 deletions apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public class HttpCallParser {
private final int sync_threshold_time;
private int sync_count = 0;
private int last_synced;
private static final LoggerMaker loggerMaker = new LoggerMaker(HttpCallParser.class);
private static final LoggerMaker loggerMaker = new LoggerMaker(HttpCallParser.class, LogDb.RUNTIME);
public APICatalogSync apiCatalogSync;
public DependencyAnalyser dependencyAnalyser;
private Map<String, Integer> hostNameToIdMap = new HashMap<>();
Expand Down Expand Up @@ -230,7 +230,11 @@ public void syncFunction(List<HttpResponseParams> responseParams, boolean syncIm

if (DbMode.dbType.equals(DbMode.DbType.MONGO_DB)) {
for (HttpResponseParams responseParam: filteredResponseParams) {
dependencyAnalyser.analyse(responseParam.getOrig(), responseParam.requestParams.getApiCollectionId());
try{
dependencyAnalyser.analyse(responseParam.getOrig(), responseParam.requestParams.getApiCollectionId());
} catch (Exception e){
loggerMaker.errorAndAddToDb(e, "error in analyzing dependency");
}
}
}

Expand Down
17 changes: 17 additions & 0 deletions apps/api-runtime/src/main/java/com/akto/utils/SampleDataToSTI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.akto.dto.traffic.SampleData;
import com.akto.dto.type.SingleTypeInfo;
Expand All @@ -13,8 +15,11 @@
import com.akto.dto.HttpResponseParams;
import com.akto.dto.SensitiveSampleData;
import com.akto.dto.type.APICatalog;
import com.akto.dto.type.RequestTemplate;
import com.akto.runtime.APICatalogSync;
import com.akto.runtime.URLAggregator;
import com.mongodb.BasicDBObject;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -124,6 +129,15 @@ private List<SingleTypeInfo> getSampleDataToSTIUtil(String dataString, String ur
return singleTypeInfos;
}

Set<String> queryParamSet = new HashSet<>();
try {
String urlWithParams = httpResponseParams.getRequestParams().getURL();
BasicDBObject queryParams = RequestTemplate.getQueryJSON(urlWithParams);
queryParamSet = new HashSet<>(queryParams.keySet());
} catch (Exception e){
logger.error(e.getMessage());
}

List<HttpResponseParams> responseParams = new ArrayList<>();
responseParams.add(httpResponseParams);
Map<Integer, URLAggregator> aggregatorMap = new HashMap<>();
Expand All @@ -140,6 +154,9 @@ private List<SingleTypeInfo> getSampleDataToSTIUtil(String dataString, String ur

for (int i = 0; i < singleTypeInfos.size(); i++) {
singleTypeInfos.get(i).setUrl(url);
if(queryParamSet.contains(singleTypeInfos.get(i).getParam())){
singleTypeInfos.get(i).setQueryParam(true);
}
}

return singleTypeInfos;
Expand Down
12 changes: 12 additions & 0 deletions libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,18 @@ public String toString() {
@BsonIgnore
private Object value;

// Only being used for generating OpenAPI spec.
@BsonIgnore
private boolean isQueryParam;

public boolean isQueryParam() {
return isQueryParam;
}

public void setQueryParam(boolean isQueryParam) {
this.isQueryParam = isQueryParam;
}

public static final String _UNIQUE_COUNT = "uniqueCount";
public long uniqueCount = 0L;
public static final String _PUBLIC_COUNT = "publicCount";
Expand Down
66 changes: 43 additions & 23 deletions libs/utils/src/main/java/com/akto/open_api/Main.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
package com.akto.open_api;

import com.akto.dao.ApiCollectionsDao;
import com.akto.dao.SingleTypeInfoDao;
import com.akto.dto.ApiCollection;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.log.LoggerMaker;
import com.akto.log.LoggerMaker.LogDb;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
import com.mongodb.client.model.Filters;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.Paths;
Expand All @@ -21,15 +17,14 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;

public class Main {
private static final LoggerMaker loggerMaker = new LoggerMaker(Main.class);
private static final LoggerMaker loggerMaker = new LoggerMaker(Main.class, LogDb.DASHBOARD);
private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static final ObjectMapper mapper = new ObjectMapper();
Expand Down Expand Up @@ -99,38 +94,63 @@ public static void addPathItems(int responseCode, Paths paths, String url, Strin
}
List<Parameter> headerParameters = new ArrayList<>();
try{
headerParameters = buildHeaders(singleTypeInfoList);
headerParameters = buildParams(singleTypeInfoList, ParamLocation.HEADER);
} catch (Exception e) {
loggerMaker.errorAndAddToDb("ERROR in building headers in addPathItems " + e, LogDb.DASHBOARD);
}

List<Parameter> queryParameters = new ArrayList<>();
try{
queryParameters = buildParams(singleTypeInfoList, ParamLocation.QUERY);
} catch (Exception e) {
loggerMaker.errorAndAddToDb("ERROR in building query params in addPathItems " + e, LogDb.DASHBOARD);
}

PathBuilder.addPathItem(paths, url, method, responseCode, schema, headerParameters, includeHeaders);
PathBuilder.addPathItem(paths, url, method, responseCode, schema, headerParameters, queryParameters, includeHeaders);
}

public static List<Parameter> buildHeaders(List<SingleTypeInfo> singleTypeInfoList) throws Exception{
List<Parameter> headerParameters = new ArrayList<>();
// Ref: https://github.com/OAI/OpenAPI-Specification/blob/3.0.1/versions/3.0.1.md#parameter-locations
public enum ParamLocation {
HEADER, QUERY, PATH, COOKIE
}

public static List<Parameter> buildParams(List<SingleTypeInfo> singleTypeInfoList, ParamLocation location) throws Exception{
List<Parameter> parameters = new ArrayList<>();
ObjectSchema schema =new ObjectSchema();
for (SingleTypeInfo singleTypeInfo: singleTypeInfoList) {
if(singleTypeInfo.isIsHeader()){
List<SchemaBuilder.CustomSchema> cc = SchemaBuilder.getCustomSchemasFromSingleTypeInfo(singleTypeInfo);
SchemaBuilder.build(schema, cc);
for (SingleTypeInfo singleTypeInfo : singleTypeInfoList) {
switch (location) {
case HEADER:
if (singleTypeInfo.isIsHeader()) {
List<SchemaBuilder.CustomSchema> cc = SchemaBuilder.getCustomSchemasFromSingleTypeInfo(singleTypeInfo);
SchemaBuilder.build(schema, cc);
}
break;
case QUERY:
if (singleTypeInfo.isQueryParam()) {
List<SchemaBuilder.CustomSchema> cc = SchemaBuilder.getCustomSchemasFromSingleTypeInfo(singleTypeInfo);
SchemaBuilder.build(schema, cc);
}
break;
default:
break;
}
}
if (schema.getProperties() == null) return headerParameters;
for(String header:schema.getProperties().keySet()){
Parameter head = new Parameter();
head.setName(header);
head.setIn("header");
head.setSchema(schema.getProperties().get(header));
headerParameters.add(head);

if (schema.getProperties() == null) return parameters;
for(String param:schema.getProperties().keySet()){
Parameter parameter = new Parameter();
parameter.setName(param);
parameter.setIn(location.name().toLowerCase());
parameter.setSchema(schema.getProperties().get(param));
parameters.add(parameter);
}
return headerParameters;
return parameters;
}

public static Schema<?> buildSchema(List<SingleTypeInfo> singleTypeInfoList) throws Exception {
ObjectSchema schema =new ObjectSchema();
for (SingleTypeInfo singleTypeInfo: singleTypeInfoList) {
if(singleTypeInfo.isIsHeader()){
if(singleTypeInfo.isIsHeader() || singleTypeInfo.isQueryParam() || singleTypeInfo.getIsUrlParam()){
continue;
}
List<SchemaBuilder.CustomSchema> cc = SchemaBuilder.getCustomSchemasFromSingleTypeInfo(singleTypeInfo);
Expand Down
7 changes: 5 additions & 2 deletions libs/utils/src/main/java/com/akto/open_api/PathBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

public class PathBuilder {

public static void addPathItem(Paths paths, String url, String method , int responseCode, Schema<?> schema,List<Parameter> headerParameters, boolean includeHeaders) throws Exception {
public static void addPathItem(Paths paths, String url, String method , int responseCode, Schema<?> schema,List<Parameter> headerParameters, List<Parameter> queryParameters, boolean includeHeaders) throws Exception {
PathItem pathItem = paths.getOrDefault(url, new PathItem());
pathItem.setDescription("description");
Operation operation = getOperation(pathItem,method);
Expand All @@ -39,9 +39,12 @@ public static void addPathItem(Paths paths, String url, String method , int resp

requestBody.setContent(requestBodyContent);
operation.setRequestBody(requestBody);
List<Parameter> parameters = new ArrayList<>();
if (includeHeaders) {
operation.setParameters(headerParameters);
parameters.addAll(headerParameters);
}
parameters.addAll(queryParameters);
operation.setParameters(parameters);
setOperation(pathItem, method, operation);
paths.addPathItem(url, pathItem);
return ;
Expand Down
4 changes: 3 additions & 1 deletion libs/utils/src/main/java/com/akto/open_api/parser/Parser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.akto.dto.HttpResponseParams;
import com.akto.dto.OriginalHttpRequest;
import com.akto.dto.OriginalHttpResponse;
import com.akto.dto.HttpResponseParams.Source;
import com.akto.dto.upload.FileUploadError;
import com.akto.dto.upload.SwaggerUploadLog;
import com.akto.log.LoggerMaker;
Expand Down Expand Up @@ -323,7 +324,8 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId
messageObject.put(mKeys.ip, "null");
messageObject.put(mKeys.time, Context.now() + "");
messageObject.put(mKeys.type, "HTTP");
messageObject.put(mKeys.source, "OTHER");
// swagger uploads are treated as HAR files.
messageObject.put(mKeys.source, Source.HAR.name());

if (responseObjectList.isEmpty()) {
responseObjectList.add(emptyResponseObject);
Expand Down

0 comments on commit 7a64f51

Please sign in to comment.