From ce48c7c7eccf94fd763ca4256b47fbfd398f8a19 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 1 Mar 2024 12:52:45 +0530 Subject: [PATCH 1/5] Moved ApiExecutor from Testing to Utils module --- .../akto/test_editor/auth/AuthValidator.java | 3 ++- .../com/akto/testing/StatusCodeAnalyser.java | 4 +-- .../ApiNodeExecutor.java | 3 ++- .../YamlNodeExecutor.java | 3 ++- .../java/com/akto/testing/ApiExecutor.java | 27 ++++++++++--------- .../com/akto/testing/HTTPClientHandler.java | 1 - .../java/com/akto/testing/HostDNSLookup.java | 0 .../src/main/java/com/akto/testing/Utils.java | 1 - 8 files changed, 23 insertions(+), 19 deletions(-) rename {apps/testing => libs/utils}/src/main/java/com/akto/testing/ApiExecutor.java (93%) rename {apps/testing => libs/utils}/src/main/java/com/akto/testing/HTTPClientHandler.java (99%) rename {apps/testing => libs/utils}/src/main/java/com/akto/testing/HostDNSLookup.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/testing/Utils.java (99%) diff --git a/apps/testing/src/main/java/com/akto/test_editor/auth/AuthValidator.java b/apps/testing/src/main/java/com/akto/test_editor/auth/AuthValidator.java index 7d13c21b29..30a9b00586 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/auth/AuthValidator.java +++ b/apps/testing/src/main/java/com/akto/test_editor/auth/AuthValidator.java @@ -11,6 +11,7 @@ import com.akto.dto.testing.TestingRunResult; import com.akto.test_editor.execution.Operations; import com.akto.testing.ApiExecutor; +import com.akto.testing.Main; import com.akto.util.CookieTransformer; import java.util.ArrayList; @@ -89,7 +90,7 @@ public static ExecutionResult checkAuth(Auth auth, RawApi rawApi, TestingRunConf OriginalHttpResponse testResponse; try { - testResponse = ApiExecutor.sendRequest(rawApi.getRequest(), true, testingRunConfig, debug, testLogs); + testResponse = ApiExecutor.sendRequest(rawApi.getRequest(), true, testingRunConfig, debug, testLogs, Main.SKIP_SSRF_CHECK); } catch(Exception e) { return new ExecutionResult(false, "error running check auth " + e.getMessage(), rawApi.getRequest(), null); } diff --git a/apps/testing/src/main/java/com/akto/testing/StatusCodeAnalyser.java b/apps/testing/src/main/java/com/akto/testing/StatusCodeAnalyser.java index a390981e36..1d5ee8d3d8 100644 --- a/apps/testing/src/main/java/com/akto/testing/StatusCodeAnalyser.java +++ b/apps/testing/src/main/java/com/akto/testing/StatusCodeAnalyser.java @@ -86,7 +86,7 @@ public static void calculateDefaultPayloads(SampleMessageStore sampleMessageStor if (idx > 0) url += "akto-"+idx; // we want to hit host url once too OriginalHttpRequest request = new OriginalHttpRequest(url, null, URLMethods.Method.GET.name(), null, new HashMap<>(), ""); - OriginalHttpResponse response = ApiExecutor.sendRequest(request, true, testingRunConfig, false, new ArrayList<>()); + OriginalHttpResponse response = ApiExecutor.sendRequest(request, true, testingRunConfig, false, new ArrayList<>(), Main.SKIP_SSRF_CHECK); boolean isStatusGood = TestPlugin.isStatusGood(response.getStatusCode()); if (!isStatusGood) continue; @@ -202,7 +202,7 @@ public static boolean fillFrequencyMap(List messages, AuthMechanism auth if (!result) return false; // execute API - OriginalHttpResponse finalResponse = ApiExecutor.sendRequest(request, true, testingRunConfig, false, new ArrayList<>()); + OriginalHttpResponse finalResponse = ApiExecutor.sendRequest(request, true, testingRunConfig, false, new ArrayList<>(), Main.SKIP_SSRF_CHECK); // if non 2xx then skip this api if (finalResponse.getStatusCode() < 200 || finalResponse.getStatusCode() >= 300) return false; diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java index 6452a5bc25..c79885d67c 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java @@ -16,6 +16,7 @@ import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.testing.ApiExecutor; +import com.akto.testing.Main; import com.akto.testing.Utils; import com.akto.utils.RedactSampleData; @@ -70,7 +71,7 @@ public NodeResult processNode(Node node, Map valuesMap, Boolean Thread.sleep(sleep); } - response = ApiExecutor.sendRequest(request, followRedirects, null, debug, testLogs); + response = ApiExecutor.sendRequest(request, followRedirects, null, debug, testLogs, Main.SKIP_SSRF_CHECK); int statusCode = response.getStatusCode(); diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index ffaea368b9..ec8f747cbf 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; +import com.akto.testing.Main; import org.json.JSONObject; import com.akto.dao.context.Context; @@ -112,7 +113,7 @@ public NodeResult processNode(Node node, Map varMap, Boolean all int tsAfterReq = 0; try { tsBeforeReq = Context.nowInMillis(); - testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs); + testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, Main.SKIP_SSRF_CHECK); tsAfterReq = Context.nowInMillis(); responseTimeArr.add(tsAfterReq - tsBeforeReq); ExecutionResult attempt = new ExecutionResult(singleReq.getSuccess(), singleReq.getErrMsg(), testReq.getRequest(), testResponse); diff --git a/apps/testing/src/main/java/com/akto/testing/ApiExecutor.java b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java similarity index 93% rename from apps/testing/src/main/java/com/akto/testing/ApiExecutor.java rename to libs/utils/src/main/java/com/akto/testing/ApiExecutor.java index 76b8b517d9..4b0c703fe7 100644 --- a/apps/testing/src/main/java/com/akto/testing/ApiExecutor.java +++ b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java @@ -30,7 +30,7 @@ public class ApiExecutor { // Load only first 1 MiB of response body into memory. private static final int MAX_RESPONSE_SIZE = 1024*1024; - private static OriginalHttpResponse common(Request request, boolean followRedirects, boolean debug, List testLogs) throws Exception { + private static OriginalHttpResponse common(Request request, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck) throws Exception { Integer accountId = Context.accountId.get(); if (accountId != null) { @@ -49,12 +49,12 @@ private static OriginalHttpResponse common(Request request, boolean followRedire if (HTTPClientHandler.instance == null) { HTTPClientHandler.initHttpClientHandler(isSaasDeployment); } - + OkHttpClient client = debug ? HTTPClientHandler.instance.getNewDebugClient(isSaasDeployment, followRedirects, testLogs) : HTTPClientHandler.instance.getHTTPClient(followRedirects); - if (!Main.SKIP_SSRF_CHECK && !HostDNSLookup.isRequestValid(request.url().host())) { + if (!skipSSRFCheck && !HostDNSLookup.isRequestValid(request.url().host())) { throw new IllegalArgumentException("SSRF attack attempt"); } @@ -149,9 +149,9 @@ public static String prepareUrl(OriginalHttpRequest request, TestingRunConfig te return replaceHostFromConfig(url, testingRunConfig); } - public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, boolean followRedirects, TestingRunConfig testingRunConfig, boolean debug, List testLogs) throws Exception { + public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, boolean followRedirects, TestingRunConfig testingRunConfig, boolean debug, List testLogs, boolean skipSSRFCheck) throws Exception { // don't lowercase url because query params will change and will result in incorrect request - + String url = prepareUrl(request, testingRunConfig); loggerMaker.infoAndAddToDb("Final url is: " + url, LogDb.TESTING); @@ -182,7 +182,7 @@ public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, bool switch (method) { case GET: case HEAD: - response = getRequest(request, builder, followRedirects, debug, testLogs); + response = getRequest(request, builder, followRedirects, debug, testLogs, skipSSRFCheck); break; case POST: case PUT: @@ -191,7 +191,7 @@ public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, bool case PATCH: case TRACK: case TRACE: - response = sendWithRequestBody(request, builder, followRedirects, debug, testLogs); + response = sendWithRequestBody(request, builder, followRedirects, debug, testLogs, skipSSRFCheck); break; case OTHER: throw new Exception("Invalid method name"); @@ -199,11 +199,14 @@ public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, bool return response; } + public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, boolean followRedirects, TestingRunConfig testingRunConfig, boolean debug, List testLogs) throws Exception { + return sendRequest(request, followRedirects, testingRunConfig, debug, testLogs, false); + } - private static OriginalHttpResponse getRequest(OriginalHttpRequest request, Request.Builder builder, boolean followRedirects, boolean debug, List testLogs) throws Exception{ + private static OriginalHttpResponse getRequest(OriginalHttpRequest request, Request.Builder builder, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck) throws Exception{ Request okHttpRequest = builder.build(); - return common(okHttpRequest, followRedirects, debug, testLogs); + return common(okHttpRequest, followRedirects, debug, testLogs, skipSSRFCheck); } public static RequestBody getFileRequestBody(String fileUrl){ @@ -248,7 +251,7 @@ public void writeTo(BufferedSink sink) throws IOException { - private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest request, Request.Builder builder, boolean followRedirects, boolean debug, List testLogs) throws Exception { + private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest request, Request.Builder builder, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck) throws Exception { Map> headers = request.getHeaders(); if (headers == null) { headers = new HashMap<>(); @@ -265,7 +268,7 @@ private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest requ Request updatedRequest = builder.build(); - return common(updatedRequest, followRedirects, debug, testLogs); + return common(updatedRequest, followRedirects, debug, testLogs, skipSSRFCheck); } String contentType = request.findContentType(); @@ -281,6 +284,6 @@ private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest requ RequestBody body = RequestBody.create(payload, MediaType.parse(contentType)); builder = builder.method(request.getMethod(), body); Request okHttpRequest = builder.build(); - return common(okHttpRequest, followRedirects, debug, testLogs); + return common(okHttpRequest, followRedirects, debug, testLogs, skipSSRFCheck); } } diff --git a/apps/testing/src/main/java/com/akto/testing/HTTPClientHandler.java b/libs/utils/src/main/java/com/akto/testing/HTTPClientHandler.java similarity index 99% rename from apps/testing/src/main/java/com/akto/testing/HTTPClientHandler.java rename to libs/utils/src/main/java/com/akto/testing/HTTPClientHandler.java index d0c42a3026..79c451c8b6 100644 --- a/apps/testing/src/main/java/com/akto/testing/HTTPClientHandler.java +++ b/libs/utils/src/main/java/com/akto/testing/HTTPClientHandler.java @@ -2,7 +2,6 @@ import com.akto.dto.RawApi; import com.akto.dto.testing.TestingRunResult; -import com.akto.rules.TestPlugin; import okhttp3.*; import okio.Buffer; import org.jetbrains.annotations.NotNull; diff --git a/apps/testing/src/main/java/com/akto/testing/HostDNSLookup.java b/libs/utils/src/main/java/com/akto/testing/HostDNSLookup.java similarity index 100% rename from apps/testing/src/main/java/com/akto/testing/HostDNSLookup.java rename to libs/utils/src/main/java/com/akto/testing/HostDNSLookup.java diff --git a/apps/testing/src/main/java/com/akto/testing/Utils.java b/libs/utils/src/main/java/com/akto/testing/Utils.java similarity index 99% rename from apps/testing/src/main/java/com/akto/testing/Utils.java rename to libs/utils/src/main/java/com/akto/testing/Utils.java index aa07320998..c20908c3a1 100644 --- a/apps/testing/src/main/java/com/akto/testing/Utils.java +++ b/libs/utils/src/main/java/com/akto/testing/Utils.java @@ -14,7 +14,6 @@ import com.akto.dto.type.RequestTemplate; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; -import com.akto.parsers.HttpCallParser; import com.akto.util.JSONUtils; import com.mongodb.BasicDBObject; From c9cfc85ae29b6798ef0b1d99c7efc902ca8e40b5 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Sat, 2 Mar 2024 15:49:46 +0530 Subject: [PATCH 2/5] Changes for swagger file --- .../java/com/akto/action/OpenApiAction.java | 227 ++++++++++++++++-- .../java/com/akto/action/PostmanAction.java | 9 +- apps/dashboard/src/main/resources/struts.xml | 40 +++ .../apps/dashboard/pages/quick_start/api.js | 14 ++ .../quick_start/components/OpenApiSource.jsx | 145 ++++++++++- .../quick_start/components/PostmanSource.jsx | 1 - .../akto/dao/upload/FileUploadLogsDao.java | 6 + .../com/akto/dao/upload/FileUploadsDao.java | 5 + .../com/akto/dto/upload/FileUploadLog.java | 10 + .../com/akto/dto/upload/PostmanUploadLog.java | 10 - .../akto/dto/upload/SwaggerFileUpload.java | 35 +++ .../com/akto/dto/upload/SwaggerUploadLog.java | 14 ++ .../java/com/akto/open_api/parser/Parser.java | 83 ++++++- .../akto/open_api/parser/ParserResult.java | 37 +++ 14 files changed, 579 insertions(+), 57 deletions(-) create mode 100644 libs/dao/src/main/java/com/akto/dto/upload/SwaggerFileUpload.java create mode 100644 libs/dao/src/main/java/com/akto/dto/upload/SwaggerUploadLog.java create mode 100644 libs/utils/src/main/java/com/akto/open_api/parser/ParserResult.java diff --git a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java index 6bfcdbee69..58ef79b15b 100644 --- a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java @@ -3,38 +3,61 @@ import com.akto.dao.ApiCollectionsDao; import com.akto.dao.SampleDataDao; import com.akto.dao.context.Context; +import com.akto.dao.file.FilesDao; +import com.akto.dao.upload.FileUploadLogsDao; +import com.akto.dao.upload.FileUploadsDao; import com.akto.dto.ApiCollection; +import com.akto.dto.files.File; import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; +import com.akto.dto.upload.*; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.open_api.Main; import com.akto.open_api.parser.Parser; +import com.akto.open_api.parser.ParserResult; import com.akto.util.Constants; +import com.akto.util.DashboardMode; +import com.akto.utils.GzipUtils; import com.akto.utils.SampleDataToSTI; import com.akto.utils.Utils; +import com.mongodb.BasicDBList; +import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.InsertManyOptions; +import com.mongodb.client.model.UpdateOptions; +import com.mongodb.client.model.Updates; +import com.mongodb.client.result.InsertOneResult; import io.swagger.parser.OpenAPIParser; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.parser.core.models.ParseOptions; import io.swagger.v3.parser.core.models.SwaggerParseResult; +import org.apache.commons.lang3.StringUtils; import org.apache.struts2.interceptor.ServletResponseAware; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class OpenApiAction extends UserAction implements ServletResponseAware { private static final LoggerMaker loggerMaker = new LoggerMaker(OpenApiAction.class); + + private final boolean skipKafka = DashboardMode.isLocalDeployment(); + private int apiCollectionId; private String openAPIString = null; private boolean includeHeaders = true; @@ -95,14 +118,26 @@ public String burpSwagger() throws IOException { } private static final String OPEN_API = "OpenAPI"; + public String importDataFromOpenApiSpec(){ int accountId = Context.accountId.get(); + + File file = new File(FileUpload.UploadType.SWAGGER_FILE.toString(), GzipUtils.zipString(openAPIString)); + InsertOneResult fileInsertId = FilesDao.instance.insertOne(file); + String fileId = fileInsertId.getInsertedId().asObjectId().toString(); + SwaggerFileUpload fileUpload = new SwaggerFileUpload(); + fileUpload.setSwaggerFileId(fileId); + fileUpload.setUploadType(FileUpload.UploadType.SWAGGER_FILE); + fileUpload.setUploadStatus(FileUpload.UploadStatus.IN_PROGRESS); + InsertOneResult insertOneResult = FileUploadsDao.instance.insertOne(fileUpload); + String fileUploadId = insertOneResult.getInsertedId().asObjectId().getValue().toString(); + this.uploadId = fileUploadId; executorService.schedule(new Runnable() { public void run() { Context.accountId.set(accountId); loggerMaker.infoAndAddToDb("Starting thread to process openAPI file", LogDb.DASHBOARD); - List messages = new ArrayList<>(); + SwaggerFileUpload fileUpload = FileUploadsDao.instance.getSwaggerMCollection().find(Filters.eq(Constants.ID, new ObjectId(fileUploadId))).first(); String title = OPEN_API + " "; try { @@ -116,26 +151,40 @@ public void run() { } else { title += Context.now(); } - messages = Parser.convertOpenApiToAkto(openAPI); - + ParserResult parsedSwagger = Parser.convertOpenApiToAkto(openAPI, fileUploadId); + List fileErrors = parsedSwagger.getFileErrors(); + + List messages = parsedSwagger.getUploadLogs(); + messages = messages.stream().peek(m -> m.setUploadId(fileUploadId)).collect(Collectors.toList()); + + int chunkSize = 100; + int numberOfChunks = (int)Math.ceil((double)messages.size() / chunkSize); + + List finalMessages = messages; + List> chunkedLists = IntStream.range(0, numberOfChunks) + .mapToObj(i -> finalMessages.subList(i * chunkSize, Math.min(finalMessages.size(), (i + 1) * chunkSize))) + .collect(Collectors.toList()); + + for(List chunk : chunkedLists){ + loggerMaker.infoAndAddToDb("Inserting chunk of size " + chunk.size(), LogDb.DASHBOARD); + FileUploadLogsDao.instance.getSwaggerMCollection().insertMany(chunk, new InsertManyOptions().ordered(true)); + } + loggerMaker.infoAndAddToDb("Inserted " + chunkedLists.size() + " chunks of logs", LogDb.DASHBOARD); + + FileUploadsDao.instance.updateOne(Filters.eq(Constants.ID, new ObjectId(fileUploadId)), Updates.combine( + Updates.set("uploadStatus", FileUpload.UploadStatus.SUCCEEDED), + Updates.set("collectionName", title), + Updates.set("errors", fileErrors), + Updates.set("count",parsedSwagger.getTotalCount()) + )); + loggerMaker.infoAndAddToDb("Finished processing openAPI file", LogDb.DASHBOARD); + } catch (Exception e) { loggerMaker.errorAndAddToDb(e, "ERROR while parsing openAPI file", LogDb.DASHBOARD); - } - - String topic = System.getenv("AKTO_KAFKA_TOPIC_NAME"); - - if (messages.size() > 0) { - apiCollectionId = title.hashCode(); - ApiCollection apiCollection = ApiCollectionsDao.instance.findOne(Filters.eq(Constants.ID, apiCollectionId)); - if (apiCollection == null) { - ApiCollectionsDao.instance.insertOne(ApiCollection.createManualCollection(apiCollectionId, title)); - } - - try { - Utils.pushDataToKafka(apiCollectionId, topic, messages, new ArrayList<>(), true); - } catch (Exception e) { - loggerMaker.errorAndAddToDb(e, "ERROR while creating collection from openAPI file", LogDb.DASHBOARD); - } + FileUploadsDao.instance.updateOne(Filters.eq(Constants.ID, new ObjectId(fileUploadId)), Updates.combine( + Updates.set("uploadStatus", FileUpload.UploadStatus.FAILED), + Updates.set("fatalError", e.getMessage()) + )); } } }, 0, TimeUnit.SECONDS); @@ -143,6 +192,138 @@ public void run() { return SUCCESS.toUpperCase(); } + private String uploadId; + + public String getUploadId() { + return uploadId; + } + + public void setUploadId(String uploadId) { + this.uploadId = uploadId; + } + + public BasicDBObject uploadDetails; + + public BasicDBObject getUploadDetails() { + return uploadDetails; + } + + public void setUploadDetails(BasicDBObject uploadDetails) { + this.uploadDetails = uploadDetails; + } + + public String fetchImportLogs(){ + if(this.uploadId == null || StringUtils.isEmpty(this.uploadId)){ + addActionError("uploadId is required"); + } + SwaggerFileUpload swaggerFileUpload = FileUploadsDao.instance.getSwaggerMCollection().find(Filters.eq(Constants.ID, new ObjectId(uploadId))).first(); + + if(swaggerFileUpload == null){ + addActionError("Invalid uploadId "); + return ERROR.toUpperCase(); + } + uploadDetails = new BasicDBObject(); + // Early return if the upload status is not SUCCEEDED + if (swaggerFileUpload.getUploadStatus() != FileUpload.UploadStatus.SUCCEEDED) { + uploadDetails.put("uploadStatus", swaggerFileUpload.getUploadStatus()); + return SUCCESS.toUpperCase(); + } + + uploadDetails.put("uploadStatus", FileUpload.UploadStatus.SUCCEEDED); + + List logs = new ArrayList<>(); + FileUploadLogsDao.instance.getSwaggerMCollection().find(Filters.eq("uploadId", uploadId)).into(logs); + + List errorLogs = logs.stream() + .filter(log -> log.getErrors() != null && !log.getErrors().isEmpty()) + .collect(Collectors.toList()); + + long apisWithErrorsAndCanBeImported = errorLogs.stream() + .filter(log -> log.getAktoFormat() != null) + .count(); + long apisWithErrorsAndCannotBeImported = errorLogs.size() - apisWithErrorsAndCanBeImported; + + BasicDBList modifiedLogs = new BasicDBList(); + if (!errorLogs.isEmpty()) { + modifiedLogs.add(new BasicDBObject("term", "Url").append("description", "Errors")); + errorLogs.forEach(log -> { + String errorDescriptions = log.getErrors().stream() + .map(FileUploadError::getError) + .collect(Collectors.joining(",")); + BasicDBObject logEntry = new BasicDBObject(); + logEntry.put("term", log.getUrl() + " (" + log.getMethod() + ")"); + logEntry.put("description", errorDescriptions); + modifiedLogs.add(logEntry); + }); + } + uploadDetails.put("collectionErrors", swaggerFileUpload.getErrors()); + uploadDetails.put("logs", modifiedLogs); + uploadDetails.put("apisWithErrorsAndParsed", apisWithErrorsAndCanBeImported); + uploadDetails.put("apisWithErrorsAndCannotBeImported", apisWithErrorsAndCannotBeImported); + uploadDetails.put("correctlyParsedApis", logs.stream().filter(log -> log.getAktoFormat() != null && (log.getErrors() == null || log.getErrors().isEmpty())).count()); + uploadDetails.put("totalCount", swaggerFileUpload.getCount()); + + return SUCCESS.toUpperCase(); + } + + public PostmanAction.ImportType importType; + public String importFile(){ + if(importType == null || uploadId == null){ + addActionError("Invalid import type or upload id"); + return ERROR.toUpperCase(); + } + SwaggerFileUpload swaggerFileUpload = FileUploadsDao.instance.getSwaggerMCollection().find(Filters.eq("_id", new ObjectId(uploadId))).first(); + if(swaggerFileUpload == null){ + addActionError("Invalid upload id"); + return ERROR.toUpperCase(); + } + List uploads = new ArrayList<>(); + + List filters = new ArrayList<>(); + filters.add(Filters.eq("uploadId", uploadId)); + filters.add(Filters.exists("aktoFormat", true)); + + if(importType == PostmanAction.ImportType.ONLY_SUCCESSFUL_APIS){ + filters.add(Filters.exists("errors", false)); + } + + FileUploadLogsDao.instance.getSwaggerMCollection().find(Filters.and(filters.toArray(new Bson[0]))).into(uploads); + int accountId = Context.accountId.get(); + + new Thread(()-> { + Context.accountId.set(accountId); + loggerMaker.infoAndAddToDb(String.format("Starting thread to import %d swagger apis, import type: %s", uploads.size(), importType), LogDb.DASHBOARD); + String topic = System.getenv("AKTO_KAFKA_TOPIC_NAME"); + + try { + String collectionId = swaggerFileUpload.getCollectionName(); + int aktoCollectionId = collectionId.hashCode(); + aktoCollectionId = aktoCollectionId < 0 ? aktoCollectionId * -1: aktoCollectionId; + List msgs = new ArrayList<>(); + loggerMaker.infoAndAddToDb(String.format("Processing swagger collection %s, aktoCollectionId: %s", swaggerFileUpload.getCollectionName(), aktoCollectionId), LogDb.DASHBOARD); + for(SwaggerUploadLog upload : uploads){ + String aktoFormat = upload.getAktoFormat(); + msgs.add(aktoFormat); + } + if(ApiCollectionsDao.instance.findOne(Filters.eq("_id", aktoCollectionId)) == null){ + String collectionName = swaggerFileUpload.getCollectionName(); + loggerMaker.infoAndAddToDb(String.format("Creating manual collection for aktoCollectionId: %s and name: %s", aktoCollectionId, collectionName), LogDb.DASHBOARD); + ApiCollectionsDao.instance.insertOne(ApiCollection.createManualCollection(aktoCollectionId, collectionName)); + } + loggerMaker.infoAndAddToDb(String.format("Pushing data in akto collection id %s", aktoCollectionId), LogDb.DASHBOARD); + Utils.pushDataToKafka(aktoCollectionId, topic, msgs, new ArrayList<>(), skipKafka); + loggerMaker.infoAndAddToDb(String.format("Pushed data in akto collection id %s", aktoCollectionId), LogDb.DASHBOARD); + FileUploadsDao.instance.getSwaggerMCollection().updateOne(Filters.eq("_id", new ObjectId(uploadId)), new BasicDBObject("$set", new BasicDBObject("ingestionComplete", true).append("markedForDeletion", true)), new UpdateOptions().upsert(false)); + loggerMaker.infoAndAddToDb("Ingestion complete for " + swaggerFileUpload.getId().toString(), LogDb.DASHBOARD); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e,"Error pushing data to kafka", LogDb.DASHBOARD); + } + }).start(); + + + return SUCCESS.toUpperCase(); + } + public void setApiCollectionId(int apiCollectionId) { this.apiCollectionId = apiCollectionId; } @@ -180,4 +361,12 @@ public String getLastFetchedMethod() { public void setLastFetchedMethod(String lastFetchedMethod) { this.lastFetchedMethod = lastFetchedMethod; } + + public PostmanAction.ImportType getImportType() { + return importType; + } + + public void setImportType(PostmanAction.ImportType importType) { + this.importType = importType; + } } diff --git a/apps/dashboard/src/main/java/com/akto/action/PostmanAction.java b/apps/dashboard/src/main/java/com/akto/action/PostmanAction.java index d35c2d315a..12c9cdeb1d 100644 --- a/apps/dashboard/src/main/java/com/akto/action/PostmanAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/PostmanAction.java @@ -36,6 +36,7 @@ import com.mongodb.client.model.Filters; import com.mongodb.client.model.ReplaceOptions; import com.mongodb.client.model.UpdateOptions; +import com.mongodb.client.model.Updates; import com.mongodb.client.result.InsertOneResult; import io.swagger.v3.oas.models.OpenAPI; import org.apache.commons.lang3.tuple.Pair; @@ -697,13 +698,13 @@ public String markImportForDeletion(){ addActionError("Upload id is required"); return ERROR.toUpperCase(); } - PostmanWorkspaceUpload postmanWorkspaceUpload = FileUploadsDao.instance.getPostmanMCollection().find(Filters.eq("_id", new ObjectId(uploadId))).first(); - if(postmanWorkspaceUpload == null){ + FileUpload id = FileUploadsDao.instance.findOne(Filters.eq("_id", new ObjectId(uploadId))); + if(id == null){ addActionError("Invalid upload id"); return ERROR.toUpperCase(); } - postmanWorkspaceUpload.setMarkedForDeletion(true); - FileUploadsDao.instance.getPostmanMCollection().updateOne(Filters.eq("_id", new ObjectId(uploadId)), new BasicDBObject("$set", new BasicDBObject("markedForDeletion", true)), new UpdateOptions().upsert(false)); + id.setMarkedForDeletion(true); + FileUploadsDao.instance.updateOneNoUpsert(Filters.eq("_id", new ObjectId(uploadId)), Updates.set("markedForDeletion", true)); return SUCCESS.toUpperCase(); } diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 8c5385f964..f7790116b9 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -532,6 +532,46 @@ + + + + + ACTIVE_ENDPOINTS + + + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + + + + + + ACTIVE_ENDPOINTS + + + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/api.js index 1d4d7ccf8d..0c2458c4a2 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/api.js @@ -100,6 +100,13 @@ const api = { data: {uploadId}, }) }, + fetchSwaggerImportLogs(uploadId){ + return request({ + url: '/api/fetchSwaggerImportLogs', + method: 'post', + data: {uploadId}, + }) + }, ingestPostman(uploadId, importType){ return request({ url: '/api/ingestPostman', @@ -107,6 +114,13 @@ const api = { data: {uploadId, importType}, }) }, + ingestSwagger(uploadId, importType){ + return request({ + url: '/api/importSwaggerLogs', + method: 'post', + data: {uploadId, importType}, + }) + }, deleteImportedPostman(uploadId){ return request({ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/OpenApiSource.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/OpenApiSource.jsx index 4b547fb782..f6460c33e4 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/OpenApiSource.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/OpenApiSource.jsx @@ -1,14 +1,21 @@ -import { HorizontalStack, Text, Badge, VerticalStack, ButtonGroup, Button, Link } from "@shopify/polaris"; +import { HorizontalStack, Text, Badge, VerticalStack, ButtonGroup, Button, Modal, DescriptionList, RadioButton, Icon, Tooltip } from "@shopify/polaris"; import { CancelMajor } from "@shopify/polaris-icons" import { useState } from "react"; import FileUpload from "../../../components/shared/FileUpload"; import api from "../api"; import func from "@/util/func"; +import SpinnerCentered from "../../../components/progress/SpinnerCentered" +import { QuestionMarkMinor } from "@shopify/polaris-icons" function OpenApiSource() { const [files, setFiles] = useState(null) const [loading, setLoading] = useState(false) + const [showImportDetailsModal, setShowImportDetailsModal] = useState(false); + const [uploadObj, setUploadObj] = useState({}) + const [importType, setImportType] = useState('ONLY_SUCCESSFUL_APIS'); + const [uploadId, setUploadId] = useState(''); + const [intervalId, setIntervalId] = useState(null); const setFilesCheck = (file) => { var reader = new FileReader() @@ -23,18 +30,23 @@ function OpenApiSource() { const formData = new FormData(); formData.append("openAPIString", files.content) await api.importDataFromOpenApiSpec(formData).then((res) => { + let uploadId = res.uploadId; setLoading(false) - let link = `/dashboard/observe/inventory/` - const forwardLink = ( - - File uploaded successfully and is being processed. You can go to - API collections - to see your APIs. - - ) - func.setToast(true, false, forwardLink); - - func.refreshApiCollections(); + setShowImportDetailsModal(true); + setUploadId(uploadId); + const id = setInterval(async () => { + api.fetchSwaggerImportLogs(uploadId).then(resp => { + if(resp.uploadDetails.uploadStatus === 'SUCCEEDED' || resp.uploadDetails.uploadStatus === 'FAILED'){ + clearInterval(id); + setIntervalId(null); + setUploadObj(resp.uploadDetails); + if(resp.uploadDetails.uploadStatus === 'FAILED'){ + setUploadId(uploadId); + } + } + }); + }, 5000); + setIntervalId(id); }).catch((err) => { setLoading(false) @@ -45,6 +57,89 @@ function OpenApiSource() { window.open("https://docs.akto.io/traffic-connections/traffic-data-sources/openapi", '_blank') } + const toggleImport = (val) => { + setImportType(val) + } + + const startImport = async() => { + setShowImportDetailsModal(false) + setUploadObj({}) + api.ingestSwagger(uploadId, importType).then(resp => { + setToast(true, false, "File import has begun, refresh inventory page to view the imported APIs") + }) + } + + const closeModal = async() => { + setShowImportDetailsModal(false) + if(intervalId != null){ + clearInterval(this.intervalId) + setIntervalId(null) + } + api.deleteImportedPostman(uploadId).then(resp => { + }) + setUploadObj({}) + } + + let successModalContent = ( +
+ Total APIs in the uploaded file: {uploadObj.totalCount} + Total apis parsed correctly by Akto: {uploadObj.correctlyParsedApis} + { + uploadObj.apisWithErrorsAndCannotBeImported + uploadObj.apisWithErrorsAndParsed > 0 && +
+ Total apis parsed with errors by Akto (can still be imported): {uploadObj.apisWithErrorsAndParsed} + Total apis which cannot be imported: {uploadObj.apisWithErrorsAndCannotBeImported} + { uploadObj.apisWithErrorsAndParsed > 0 && +
+
+ toggleImport("ALL_APIS")} /> +
+ + + +
+
+
+ toggleImport("ONLY_SUCCESSFUL_APIS")}/> +
+ + + +
+
+
+ } +
+ } + {uploadObj.collectionErrors > 0 && +
+ {uploadObj.collectionErrors.forEach(error => { + {{error}} + })} +
+ } + {uploadObj.collectionErrors && uploadObj.collectionErrors.length > 0 && +
+ File level errors + +
+ } + + {uploadObj.logs && uploadObj.logs.length > 0 && +
+ Url level errors + +
+ } + + + +
+ ) return (
@@ -78,6 +173,32 @@ function OpenApiSource() { + {closeModal()}} + title="Swagger Import details" + key="swagger-import-details-modal" + primaryAction={{ + content: "Import", + onAction: startImport, + disabled: uploadObj.uploadStatus !== 'SUCCEEDED' || ( uploadObj.uploadStatus === 'SUCCEEDED' && (uploadObj.apisWithErrorsAndParsed + uploadObj.correctlyParsedApis === 0)) + }} + secondaryActions={[ + { + content: 'Cancel', + onAction: closeModal, + }, + ]} + > + + {uploadObj.uploadStatus === 'SUCCEEDED' ? + successModalContent + : uploadObj.uploadStatus === 'FAILED' ? + Something went wrong while processing this postman import. Please reach out to us at help@akto.io with the following uploadId: {uploadId} for further support. : +
We are analyzing the swagger file before we import it in Akto
+ } + +
) } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/PostmanSource.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/PostmanSource.jsx index 91806cb710..922c83e939 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/PostmanSource.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/components/PostmanSource.jsx @@ -204,7 +204,6 @@ function PostmanSource() { } const closeModal = async() => { - console.log("I am called") setShowImportDetailsModal(false) if(intervalId != null){ clearInterval(this.intervalId) diff --git a/libs/dao/src/main/java/com/akto/dao/upload/FileUploadLogsDao.java b/libs/dao/src/main/java/com/akto/dao/upload/FileUploadLogsDao.java index ff3c9c2432..a1f266b4f5 100644 --- a/libs/dao/src/main/java/com/akto/dao/upload/FileUploadLogsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/upload/FileUploadLogsDao.java @@ -3,6 +3,7 @@ import com.akto.dao.AccountsContextDao; import com.akto.dto.upload.FileUploadLog; import com.akto.dto.upload.PostmanUploadLog; +import com.akto.dto.upload.SwaggerUploadLog; import com.mongodb.client.MongoCollection; public class FileUploadLogsDao extends AccountsContextDao { @@ -21,4 +22,9 @@ public Class getClassT() { public MongoCollection getPostmanMCollection() { return getMCollection(getDBName(), getCollName(), PostmanUploadLog.class); } + public MongoCollection getSwaggerMCollection() { + return getMCollection(getDBName(), getCollName(), SwaggerUploadLog.class); + } + + } diff --git a/libs/dao/src/main/java/com/akto/dao/upload/FileUploadsDao.java b/libs/dao/src/main/java/com/akto/dao/upload/FileUploadsDao.java index 6e363cbdde..e05df62f36 100644 --- a/libs/dao/src/main/java/com/akto/dao/upload/FileUploadsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/upload/FileUploadsDao.java @@ -3,6 +3,7 @@ import com.akto.dao.AccountsContextDao; import com.akto.dto.upload.FileUpload; import com.akto.dto.upload.PostmanWorkspaceUpload; +import com.akto.dto.upload.SwaggerFileUpload; import com.mongodb.client.MongoCollection; public class FileUploadsDao extends AccountsContextDao { @@ -18,6 +19,10 @@ public Class getClassT() { public static final FileUploadsDao instance = new FileUploadsDao(); + public MongoCollection getSwaggerMCollection() { + return getMCollection(getDBName(), getCollName(), SwaggerFileUpload.class); + } + public MongoCollection getPostmanMCollection() { return getMCollection(getDBName(), getCollName(), PostmanWorkspaceUpload.class); } diff --git a/libs/dao/src/main/java/com/akto/dto/upload/FileUploadLog.java b/libs/dao/src/main/java/com/akto/dto/upload/FileUploadLog.java index 1d9f5e4878..ba0aa46dfe 100644 --- a/libs/dao/src/main/java/com/akto/dto/upload/FileUploadLog.java +++ b/libs/dao/src/main/java/com/akto/dto/upload/FileUploadLog.java @@ -24,6 +24,16 @@ public enum UploadLogStatus { protected String url; + private String aktoFormat; + + public String getAktoFormat() { + return aktoFormat; + } + + public void setAktoFormat(String aktoFormat) { + this.aktoFormat = aktoFormat; + } + public ObjectId getId() { return id; } diff --git a/libs/dao/src/main/java/com/akto/dto/upload/PostmanUploadLog.java b/libs/dao/src/main/java/com/akto/dto/upload/PostmanUploadLog.java index aea156c1f2..770b0533f1 100644 --- a/libs/dao/src/main/java/com/akto/dto/upload/PostmanUploadLog.java +++ b/libs/dao/src/main/java/com/akto/dto/upload/PostmanUploadLog.java @@ -7,8 +7,6 @@ public PostmanUploadLog() { private String postmanWorkspaceId; private String postmanCollectionId; - private String aktoFormat; - public String getPostmanWorkspaceId() { return postmanWorkspaceId; } @@ -24,12 +22,4 @@ public void setPostmanWorkspaceId(String postmanWorkspaceId) { public void setPostmanCollectionId(String postmanCollectionId) { this.postmanCollectionId = postmanCollectionId; } - - public String getAktoFormat() { - return aktoFormat; - } - - public void setAktoFormat(String aktoFormat) { - this.aktoFormat = aktoFormat; - } } diff --git a/libs/dao/src/main/java/com/akto/dto/upload/SwaggerFileUpload.java b/libs/dao/src/main/java/com/akto/dto/upload/SwaggerFileUpload.java new file mode 100644 index 0000000000..6859bf4b87 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/upload/SwaggerFileUpload.java @@ -0,0 +1,35 @@ +package com.akto.dto.upload; + +import java.util.List; + +public class SwaggerFileUpload extends FileUpload{ + + private String swaggerFileId; + private String collectionName; + + private List errors; + + public String getSwaggerFileId() { + return swaggerFileId; + } + + public void setSwaggerFileId(String swaggerFileId) { + this.swaggerFileId = swaggerFileId; + } + + public String getCollectionName() { + return collectionName; + } + + public void setCollectionName(String collectionName) { + this.collectionName = collectionName; + } + + public List getErrors() { + return errors; + } + + public void setErrors(List errors) { + this.errors = errors; + } +} diff --git a/libs/dao/src/main/java/com/akto/dto/upload/SwaggerUploadLog.java b/libs/dao/src/main/java/com/akto/dto/upload/SwaggerUploadLog.java new file mode 100644 index 0000000000..f5239b91cb --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/upload/SwaggerUploadLog.java @@ -0,0 +1,14 @@ +package com.akto.dto.upload; + +public class SwaggerUploadLog extends FileUploadLog{ + + private String method; + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } +} diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java b/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java index 827a79f912..1f2359c414 100644 --- a/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java +++ b/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java @@ -1,20 +1,21 @@ package com.akto.open_api.parser; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import javax.ws.rs.core.Response.Status; import com.akto.dao.context.Context; import com.akto.dto.HttpResponseParams; +import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.upload.FileUploadError; +import com.akto.dto.upload.SwaggerUploadLog; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.open_api.parser.parameter_parser.CookieParser; import com.akto.open_api.parser.parameter_parser.HeaderParser; import com.akto.open_api.parser.parameter_parser.PathParamParser; import com.akto.open_api.parser.parameter_parser.QueryParamParser; +import com.akto.testing.ApiExecutor; import com.akto.util.Pair; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -31,18 +32,21 @@ import io.swagger.v3.oas.models.servers.Server; import io.swagger.v3.parser.util.ResolverFully; +import static com.akto.dto.RawApi.convertHeaders; + public class Parser { private static final LoggerMaker loggerMaker = new LoggerMaker(Parser.class, LogDb.DASHBOARD); private static final ObjectMapper mapper = new ObjectMapper(); - public static List convertOpenApiToAkto(OpenAPI openAPI) { + public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId) { + List fileLevelErrors = new ArrayList<>(); + List uploadLogs = new ArrayList<>(); // replaces all refs with actual objects from components. ResolverFully resolverUtil = new ResolverFully(); resolverUtil.resolveFully(openAPI); - List messages = new ArrayList<>(); Paths paths = openAPI.getPaths(); @@ -83,6 +87,7 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { } } catch (Exception e) { loggerMaker.infoAndAddToDb("unable to parse security schemes " + e.getMessage()); + fileLevelErrors.add(new FileUploadError("unable to parse security schemes " + e.getMessage(), FileUploadError.ErrorType.WARNING)); } List rootSecurity = openAPI.getSecurity(); @@ -90,6 +95,7 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { rootSecurity = new ArrayList<>(); } + int count = 0; for (String path : paths.keySet()) { String originalPath = String.copyValueOf(path.toCharArray()); PathItem pathItem = paths.get(path); @@ -101,8 +107,9 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { List serversFromPath = pathItem.getServers(); List parametersFromPath = pathItem.getParameters(); - for (PathItem.HttpMethod operationType : pathItem.readOperationsMap().keySet()) { - PathItem.HttpMethod method = operationType; + for (PathItem.HttpMethod method : pathItem.readOperationsMap().keySet()) { + count++; + List apiLevelErrors = new ArrayList<>(); Operation operation = pathItem.readOperationsMap().get(method); List parametersFromOperation = operation.getParameters(); @@ -146,6 +153,7 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { } catch (Exception e) { loggerMaker.infoAndAddToDb( "unable to handle request body for " + path + " " + method + " " + e.toString()); + apiLevelErrors.add(new FileUploadError("Unable to parse request body: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); } List operationSecurity = operation.getSecurity(); @@ -187,6 +195,7 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { } } catch (Exception e) { loggerMaker.infoAndAddToDb("unable to parse security schemes " + e.getMessage()); + apiLevelErrors.add(new FileUploadError("unable to parse security schemes: " + e.getMessage(), FileUploadError.ErrorType.WARNING)); } String requestHeadersString = ""; @@ -197,6 +206,7 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { loggerMaker.infoAndAddToDb( "unable to parse request headers for " + path + " " + method + " " + e.getMessage()); + apiLevelErrors.add(new FileUploadError("error while converting request headers to string: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); } } @@ -249,6 +259,7 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { } catch (Exception e) { loggerMaker.infoAndAddToDb("unable to handle response headers for " + path + " " + method + " " + e.toString()); + apiLevelErrors.add(new FileUploadError("Error while converting response headers to string: " +e.getMessage(),FileUploadError.ErrorType.ERROR)); } responseObject.put(mKeys.responsePayload, responseString); @@ -257,10 +268,44 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { } } } + else { + loggerMaker.infoAndAddToDb("no responses found for " + path + " " + method + ", replaying request"); + + Map> modifiedHeaders = new HashMap<>(); + for (String key : requestHeaders.keySet()) { + modifiedHeaders.put(key, Collections.singletonList(requestHeaders.get(key))); + } + OriginalHttpRequest originalHttpRequest = new OriginalHttpRequest(path, "", method.toString(), requestString, modifiedHeaders, "http"); + String responseHeadersString; + String responsePayload; + String statusCode; + String status; + try { + OriginalHttpResponse res = ApiExecutor.sendRequest(originalHttpRequest, true, null, false, new ArrayList<>()); + responseHeadersString = convertHeaders(res.getHeaders()); + responsePayload = res.getBody(); + statusCode = res.getStatusCode()+""; + status = ""; + if(res.getStatusCode() < 200 || res.getStatusCode() >=400){ + throw new Exception("Found non 2XX response on replaying the API"); + } + Map responseObject = new HashMap<>(); + responseObject.put(mKeys.responsePayload, responsePayload); + responseObject.put(mKeys.responseHeaders, responseHeadersString); + responseObject.put(mKeys.status, status); + responseObject.put(mKeys.statusCode, statusCode); + responseObjectList.add(responseObject); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e,"Error while making request for " + originalHttpRequest.getFullUrlWithParams() + " : " + e.toString(), LogDb.DASHBOARD); + apiLevelErrors.add(new FileUploadError("Error while making request, reason: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); + } + + } } catch (Exception e) { loggerMaker.infoAndAddToDb( "unable to handle response body for " + path + " " + method + " " + e.toString()); + apiLevelErrors.add(new FileUploadError("Error while converting response to Akto format: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); } messageObject.put(mKeys.akto_account_id, Context.accountId.get().toString()); @@ -287,13 +332,25 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { */ fillDummyIfEmptyMessage(messageObject); + SwaggerUploadLog log = new SwaggerUploadLog(); + log.setMethod(method.toString()); + log.setUrl(path); + log.setUploadId(uploadId); + try { String s = mapper.writeValueAsString(messageObject); - messages.add(s); + log.setAktoFormat(s); + } catch (JsonProcessingException e) { loggerMaker.infoAndAddToDb("unable to parse message object for " + path + " " + method + " " + e.getMessage()); + apiLevelErrors.add(new FileUploadError("Error while converting message to Akto format: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); + log.setAktoFormat(null); + } + if(!apiLevelErrors.isEmpty()) { + log.setErrors(apiLevelErrors); } + uploadLogs.add(log); } } @@ -302,7 +359,11 @@ public static List convertOpenApiToAkto(OpenAPI openAPI) { } } - return messages; + ParserResult parserResult = new ParserResult(); + parserResult.setFileErrors(fileLevelErrors); + parserResult.setUploadLogs(uploadLogs); + parserResult.setTotalCount(count); + return parserResult; } // message keys for akto format. diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/ParserResult.java b/libs/utils/src/main/java/com/akto/open_api/parser/ParserResult.java new file mode 100644 index 0000000000..68e6ae10cf --- /dev/null +++ b/libs/utils/src/main/java/com/akto/open_api/parser/ParserResult.java @@ -0,0 +1,37 @@ +package com.akto.open_api.parser; + +import com.akto.dto.upload.FileUploadError; +import com.akto.dto.upload.SwaggerUploadLog; + +import java.util.List; + +public class ParserResult { + private int totalCount; + private List fileErrors; + + private List uploadLogs; + + public int getTotalCount() { + return totalCount; + } + + public void setTotalCount(int totalCount) { + this.totalCount = totalCount; + } + + public List getFileErrors() { + return fileErrors; + } + + public void setFileErrors(List fileErrors) { + this.fileErrors = fileErrors; + } + + public List getUploadLogs() { + return uploadLogs; + } + + public void setUploadLogs(List uploadLogs) { + this.uploadLogs = uploadLogs; + } +} From 79058504ed1500a385df503126b36b4e8715c32c Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 4 Mar 2024 11:25:26 +0530 Subject: [PATCH 3/5] Fixed NPE --- .../open_api/parser/parameter_parser/QueryParamParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/QueryParamParser.java b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/QueryParamParser.java index 33eaf2df11..8fe23a0afd 100644 --- a/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/QueryParamParser.java +++ b/libs/utils/src/main/java/com/akto/open_api/parser/parameter_parser/QueryParamParser.java @@ -57,8 +57,8 @@ private static String addQueryParametersUtil(Parameter parameter) { String existingExample = CommonParser.getExistingExample(parameter); if (existingExample != null) { ret = existingExample; - } else if (parameter.getStyle().equals(Parameter.StyleEnum.FORM) && - parameter.getExplode().equals(true) && + } else if (Parameter.StyleEnum.FORM.equals(parameter.getStyle()) && + parameter.getExplode() && parameter.getSchema() != null) { Example example = ExampleBuilder.fromSchema(parameter.getSchema(), null); From e1fe04d89ebda77cb5b9dc351d48dfe24c456fec Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 4 Mar 2024 14:45:07 +0530 Subject: [PATCH 4/5] Added changes to replay request and then fallback on saved messages --- .../java/com/akto/open_api/parser/Parser.java | 177 +++++++++--------- 1 file changed, 92 insertions(+), 85 deletions(-) diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java b/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java index 1f2359c414..708fa0504f 100644 --- a/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java +++ b/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java @@ -120,22 +120,28 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId * PATH parameters take precedence over ROOT parameters. * OPERATION parameters take precedence over PATH parameters. */ - path = PathParamParser.replacePathParameter(originalPath, Arrays.asList( - parametersFromOperation, parametersFromPath)); + Map requestHeaders = new HashMap<>(); + try { + path = PathParamParser.replacePathParameter(originalPath, Arrays.asList( + parametersFromOperation, parametersFromPath)); - path = QueryParamParser.addQueryParameters(path, Arrays.asList( - parametersFromPath, parametersFromOperation)); + path = QueryParamParser.addQueryParameters(path, Arrays.asList( + parametersFromPath, parametersFromOperation)); - path = ServerParser.addServer(path, Arrays.asList( - serversFromOperation, serversFromPath, servers)); + path = ServerParser.addServer(path, Arrays.asList( + serversFromOperation, serversFromPath, servers)); - Map requestHeaders = HeaderParser.buildHeaders( - Arrays.asList(parametersFromPath, parametersFromOperation)); + requestHeaders = HeaderParser.buildHeaders( + Arrays.asList(parametersFromPath, parametersFromOperation)); - Map cookieHeaders = CookieParser.getCookieHeader( - Arrays.asList(parametersFromPath, parametersFromOperation)); + Map cookieHeaders = CookieParser.getCookieHeader( + Arrays.asList(parametersFromPath, parametersFromOperation)); - requestHeaders.putAll(cookieHeaders); + requestHeaders.putAll(cookieHeaders); + } catch (Exception e) { + loggerMaker.infoAndAddToDb("unable to parse path parameters for " + path + " " + method + " " + e.toString()); + apiLevelErrors.add(new FileUploadError("unable to parse path parameters: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); + } String requestString = ""; try { @@ -215,97 +221,98 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId List> responseObjectList = new ArrayList<>(); try { - ApiResponses responses = operation.getResponses(); - - if (responses != null) { - for (String responseCode : responses.keySet()) { - - if (responseCode.equals(ApiResponses.DEFAULT)) - continue; - - int statusCode = Integer.parseInt(responseCode); - if (HttpResponseParams.validHttpResponseCode(statusCode)) { + loggerMaker.infoAndAddToDb("no responses found for " + path + " " + method + ", replaying request"); - String responseString = ""; - String responseHeadersString = ""; - Map responseObject = new HashMap<>(); - - ApiResponse response = responses.get(responseCode); + Map> modifiedHeaders = new HashMap<>(); + for (String key : requestHeaders.keySet()) { + modifiedHeaders.put(key, Collections.singletonList(requestHeaders.get(key))); + } + OriginalHttpRequest originalHttpRequest = new OriginalHttpRequest(path, "", method.toString(), requestString, modifiedHeaders, "http"); + String responseHeadersString; + String responsePayload; + String statusCode; + String status; + try { + OriginalHttpResponse res = ApiExecutor.sendRequest(originalHttpRequest, true, null, false, new ArrayList<>()); + responseHeadersString = convertHeaders(res.getHeaders()); + responsePayload = res.getBody(); + statusCode = res.getStatusCode()+""; + status = ""; + if(res.getStatusCode() < 200 || res.getStatusCode() >=400){ + throw new Exception("Found non 2XX response on replaying the API"); + } + Map responseObject = new HashMap<>(); + responseObject.put(mKeys.responsePayload, responsePayload); + responseObject.put(mKeys.responseHeaders, responseHeadersString); + responseObject.put(mKeys.status, status); + responseObject.put(mKeys.statusCode, statusCode); + responseObjectList.add(responseObject); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e,"Error while making request for " + originalHttpRequest.getFullUrlWithParams() + " : " + e.getMessage(), LogDb.DASHBOARD); + ApiResponses responses = operation.getResponses(); + if (responses != null) { + for (String responseCode : responses.keySet()) { + + if (responseCode.equals(ApiResponses.DEFAULT)) + continue; + + int statusCodeInt = Integer.parseInt(responseCode); + if (HttpResponseParams.validHttpResponseCode(statusCodeInt)) { + + String responseString = ""; + responseHeadersString = ""; + Map responseObject = new HashMap<>(); + + ApiResponse response = responses.get(responseCode); + + responseObject.put(mKeys.statusCode, responseCode); + Status statusObj = Status.fromStatusCode(statusCodeInt); + if (statusObj != null) { + responseObject.put(mKeys.status, statusObj.getReasonPhrase()); + } - responseObject.put(mKeys.statusCode, responseCode); - Status status = Status.fromStatusCode(statusCode); - if (status != null) { - responseObject.put(mKeys.status, status.getReasonPhrase()); - } + Content content = response.getContent(); - Content content = response.getContent(); + Map headers = response.getHeaders(); + Map responseHeaders = new HashMap<>(); + if (headers != null) { + responseHeaders = HeaderParser.buildResponseHeaders(headers); + } - Map headers = response.getHeaders(); - Map responseHeaders = new HashMap<>(); - if (headers != null) { - responseHeaders = HeaderParser.buildResponseHeaders(headers); - } + if (content != null) { + Pair example = ContentParser.getExampleFromContent(content); + if (!(example.getFirst().isEmpty())) { + responseHeaders.put("Content-Type", example.getFirst()); + } + responseString = example.getSecond(); + } - if (content != null) { - Pair example = ContentParser.getExampleFromContent(content); - if (!(example.getFirst().isEmpty())) { - responseHeaders.put("Content-Type", example.getFirst()); + try { + responseHeadersString = mapper.writeValueAsString(responseHeaders); + } catch (Exception e1) { + loggerMaker.infoAndAddToDb("unable to handle response headers for " + path + " " + + method + " " + e1.getMessage()); + apiLevelErrors.add(new FileUploadError("Replaying the request failed, reason: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); + apiLevelErrors.add(new FileUploadError("Error while converting response headers to string from example: " +e1.getMessage(),FileUploadError.ErrorType.ERROR)); } - responseString = example.getSecond(); - } - try { - responseHeadersString = mapper.writeValueAsString(responseHeaders); - } catch (Exception e) { - loggerMaker.infoAndAddToDb("unable to handle response headers for " + path + " " - + method + " " + e.toString()); - apiLevelErrors.add(new FileUploadError("Error while converting response headers to string: " +e.getMessage(),FileUploadError.ErrorType.ERROR)); + responseObject.put(mKeys.responsePayload, responseString); + responseObject.put(mKeys.responseHeaders, responseHeadersString); + responseObjectList.add(responseObject); } - - responseObject.put(mKeys.responsePayload, responseString); - responseObject.put(mKeys.responseHeaders, responseHeadersString); - responseObjectList.add(responseObject); } } - } - else { - loggerMaker.infoAndAddToDb("no responses found for " + path + " " + method + ", replaying request"); - - Map> modifiedHeaders = new HashMap<>(); - for (String key : requestHeaders.keySet()) { - modifiedHeaders.put(key, Collections.singletonList(requestHeaders.get(key))); - } - OriginalHttpRequest originalHttpRequest = new OriginalHttpRequest(path, "", method.toString(), requestString, modifiedHeaders, "http"); - String responseHeadersString; - String responsePayload; - String statusCode; - String status; - try { - OriginalHttpResponse res = ApiExecutor.sendRequest(originalHttpRequest, true, null, false, new ArrayList<>()); - responseHeadersString = convertHeaders(res.getHeaders()); - responsePayload = res.getBody(); - statusCode = res.getStatusCode()+""; - status = ""; - if(res.getStatusCode() < 200 || res.getStatusCode() >=400){ - throw new Exception("Found non 2XX response on replaying the API"); - } - Map responseObject = new HashMap<>(); - responseObject.put(mKeys.responsePayload, responsePayload); - responseObject.put(mKeys.responseHeaders, responseHeadersString); - responseObject.put(mKeys.status, status); - responseObject.put(mKeys.statusCode, statusCode); - responseObjectList.add(responseObject); - } catch (Exception e) { - loggerMaker.errorAndAddToDb(e,"Error while making request for " + originalHttpRequest.getFullUrlWithParams() + " : " + e.toString(), LogDb.DASHBOARD); - apiLevelErrors.add(new FileUploadError("Error while making request, reason: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); + else { + apiLevelErrors.add(new FileUploadError("Replaying the request failed, reason: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); + apiLevelErrors.add(new FileUploadError("No example responses found for the API in the uploaded file", FileUploadError.ErrorType.ERROR)); } - } } catch (Exception e) { loggerMaker.infoAndAddToDb( "unable to handle response body for " + path + " " + method + " " + e.toString()); apiLevelErrors.add(new FileUploadError("Error while converting response to Akto format: " + e.getMessage(), FileUploadError.ErrorType.ERROR)); + } messageObject.put(mKeys.akto_account_id, Context.accountId.get().toString()); From d12e57436ec67dbcffe3d116898fe606826d8182 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 4 Mar 2024 17:37:27 +0530 Subject: [PATCH 5/5] Fixed log --- libs/utils/src/main/java/com/akto/open_api/parser/Parser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java b/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java index 708fa0504f..30b29a7929 100644 --- a/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java +++ b/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java @@ -221,7 +221,7 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId List> responseObjectList = new ArrayList<>(); try { - loggerMaker.infoAndAddToDb("no responses found for " + path + " " + method + ", replaying request"); + loggerMaker.infoAndAddToDb("Replaying request for" + path + " " + method + ", replaying request"); Map> modifiedHeaders = new HashMap<>(); for (String key : requestHeaders.keySet()) { @@ -238,7 +238,7 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId responsePayload = res.getBody(); statusCode = res.getStatusCode()+""; status = ""; - if(res.getStatusCode() < 200 || res.getStatusCode() >=400){ + if(!HttpResponseParams.validHttpResponseCode(res.getStatusCode())){ throw new Exception("Found non 2XX response on replaying the API"); } Map responseObject = new HashMap<>();