Skip to content

Commit

Permalink
insert testing run results in smaller batches
Browse files Browse the repository at this point in the history
  • Loading branch information
@notshivansh
notshivansh committed Nov 11, 2023
1 parent 866701e commit d49fd01
Showing 1 changed file with 27 additions and 36 deletions.
63 changes: 27 additions & 36 deletions apps/testing/src/main/java/com/akto/testing/TestExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {

CountDownLatch latch = new CountDownLatch(apiInfoKeyList.size());
ExecutorService threadPool = Executors.newFixedThreadPool(maxConcurrentRequests);
List<Future<List<TestingRunResult>>> futureTestingRunResults = new ArrayList<>();
List<Future<Void>> futureTestingRunResults = new ArrayList<>();
Map<String, Integer> hostsToApiCollectionMap = new HashMap<>();

ConcurrentHashMap<String, String> subCategoryEndpointMap = new ConcurrentHashMap<>();
Expand Down Expand Up @@ -200,7 +200,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {
loggerMaker.errorAndAddToDb("Error while finding host: " + e, LogDb.TESTING);
}
try {
Future<List<TestingRunResult>> future = threadPool.submit(
Future<Void> future = threadPool.submit(
() -> startWithLatch(apiInfoKey,
testingRun.getTestIdConfig(),
testingRun.getId(),testingRun.getTestingRunConfig(), testingUtil, summaryId,
Expand All @@ -225,20 +225,6 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {

loggerMaker.infoAndAddToDb("Finished testing", LogDb.TESTING);

int totalResults = 0;
for (Future<List<TestingRunResult>> future: futureTestingRunResults) {
if (!future.isDone()) continue;
try {
if (!future.get().isEmpty()) {
int resultSize = future.get().size();
totalResults += resultSize;
}
} catch (InterruptedException | ExecutionException e) {
loggerMaker.errorAndAddToDb("Error while after running test : " + e, LogDb.TESTING);
}
}

loggerMaker.infoAndAddToDb("Finished adding " + totalResults + " testingRunResults", LogDb.TESTING);
}

public static void updateTestSummary(ObjectId summaryId){
Expand Down Expand Up @@ -463,7 +449,7 @@ public Map<String, Object> generateResponseMap(String payloadStr, Map<String, Li
return respMap;
}

public List<TestingRunResult> startWithLatch(
public Void startWithLatch(
ApiInfo.ApiInfoKey apiInfoKey, int testIdConfig, ObjectId testRunId, TestingRunConfig testingRunConfig,
TestingUtil testingUtil, ObjectId testRunResultSummaryId, int accountId, CountDownLatch latch, int startTime,
int timeToKill, Map<String, TestConfig> testConfigMap, TestingRun testingRun,
Expand All @@ -472,35 +458,20 @@ public List<TestingRunResult> startWithLatch(
loggerMaker.infoAndAddToDb("Starting test for " + apiInfoKey, LogDb.TESTING);

Context.accountId.set(accountId);
List<TestingRunResult> testingRunResults = new ArrayList<>();
int now = Context.now();
if ( timeToKill <= 0 || now - startTime <= timeToKill) {
try {
// todo: commented out older one
// testingRunResults = start(apiInfoKey, testIdConfig, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap);
testingRunResults = startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap);
String size = testingRunResults.size()+"";
loggerMaker.infoAndAddToDb("testingRunResults size: " + size, LogDb.TESTING);
if (!testingRunResults.isEmpty()) {
trim(testingRunResults);
TestingRunResultDao.instance.insertMany(testingRunResults);
loggerMaker.infoAndAddToDb("Inserted testing results", LogDb.TESTING);
//Creating issues from testingRunResults
TestingIssuesHandler handler = new TestingIssuesHandler();
boolean triggeredByTestEditor = false;
if (testingRun.getTriggeredBy() != null) {
triggeredByTestEditor = testingRun.getTriggeredBy().equals("test_editor");
}
handler.handleIssuesCreationFromTestingRunResults(testingRunResults, triggeredByTestEditor); // pass new field here
}
startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap);
} catch (Exception e) {
e.printStackTrace();
loggerMaker.errorAndAddToDb("error while running tests: " + e, LogDb.TESTING);
}
}

latch.countDown();
return testingRunResults;
return null;
}

public static void trim(TestingRunResult testingRunResult) {
Expand Down Expand Up @@ -535,14 +506,27 @@ public void trim(List<TestingRunResult> testingRunResults) {
}
}

public List<TestingRunResult> startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
public void insertResultsAndMakeIssues(List<TestingRunResult> testingRunResults) {
String size = testingRunResults.size() + "";
loggerMaker.infoAndAddToDb("testingRunResults size: " + size, LogDb.TESTING);
trim(testingRunResults);
TestingRunResultDao.instance.insertMany(testingRunResults);
loggerMaker.infoAndAddToDb("Inserted testing results", LogDb.TESTING);
TestingIssuesHandler handler = new TestingIssuesHandler();
boolean triggeredByTestEditor = false;
handler.handleIssuesCreationFromTestingRunResults(testingRunResults, triggeredByTestEditor);
testingRunResults.clear();
}

public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
TestingRunConfig testingRunConfig, TestingUtil testingUtil,
ObjectId testRunResultSummaryId, Map<String, TestConfig> testConfigMap,
ConcurrentHashMap<String, String> subCategoryEndpointMap, Map<ApiInfoKey, String> apiInfoKeyToHostMap) {
List<TestingRunResult> testingRunResults = new ArrayList<>();

List<String> testSubCategories = testingRunConfig == null ? new ArrayList<>() : testingRunConfig.getTestSubCategoryList();

int random = (int) ( ( Math.random() * 10 ) + 4);
for (String testSubCategory: testSubCategories) {
TestConfig testConfig = testConfigMap.get(testSubCategory);
if (testConfig == null) continue;
Expand All @@ -557,9 +541,16 @@ public List<TestingRunResult> startTestNew(ApiInfo.ApiInfoKey apiInfoKey, Object
e.printStackTrace();
}
if (testingRunResult != null) testingRunResults.add(testingRunResult);

if (!testingRunResults.isEmpty() && testingRunResults.size() % random == 0) {
insertResultsAndMakeIssues(testingRunResults);
}
}

if(!testingRunResults.isEmpty()){
insertResultsAndMakeIssues(testingRunResults);
}

return testingRunResults;
}

public boolean applyRunOnceCheck(ApiInfoKey apiInfoKey, TestConfig testConfig, ConcurrentHashMap<String, String> subCategoryEndpointMap, Map<ApiInfoKey, String> apiInfoKeyToHostMap, String testSubCategory) {
Expand Down

0 comments on commit d49fd01

Please sign in to comment.