diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java index 92c5b479..b3de2c4e 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java @@ -26,6 +26,7 @@ import io.dockstore.common.metrics.MetricsData; import io.dockstore.common.metrics.MetricsDataS3Client; import io.dockstore.openapi.client.api.ExtendedGa4GhApi; +import io.dockstore.openapi.client.model.AggregatedExecution; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; @@ -108,7 +109,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) { if (!allMetrics.isEmpty()) { // Calculate metrics across all platforms by aggregating the aggregated metrics from each platform try { - getAggregatedMetrics(new ExecutionsRequestBody().aggregatedExecutions(allMetrics)).ifPresent(metrics -> { + getAggregatedMetrics(allMetrics).ifPresent(metrics -> { extendedGa4GhApi.aggregatedMetricsPut(metrics, Partner.ALL.name(), toolId, versionName); System.out.printf("Aggregated metrics across all platforms (%s) for tool ID %s, version %s from directory %s%n", platformsString, toolId, versionName, versionS3KeyPrefix); @@ -135,7 +136,7 @@ private ExecutionsRequestBody getExecutions(String toolId, String versionName, S List runExecutionsFromAllSubmissions = new ArrayList<>(); List taskExecutionsFromAllSubmissions = new ArrayList<>(); List validationExecutionsFromAllSubmissions = new ArrayList<>(); - List aggregatedExecutionsFromAllSubmissions = new ArrayList<>(); + List aggregatedExecutionsFromAllSubmissions = new ArrayList<>(); for (MetricsData metricsData : metricsDataList) { String fileContent = metricsDataS3Client.getMetricsDataFileContent(metricsData.toolId(), metricsData.toolVersionName(), diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java index 49036119..978b23ea 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java @@ -40,6 +40,8 @@ import java.nio.file.Files; import java.util.List; import java.util.Optional; +import java.util.UUID; + import org.apache.commons.configuration2.INIConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,6 +186,7 @@ private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolE .validatorToolVersion(validatorVersion) .isValid(isValid); validationExecution.setDateExecuted(dateExecuted); + validationExecution.setExecutionId(UUID.randomUUID().toString()); // No execution ID was provided by DNAstack, generate a random one ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().validationExecutions(List.of(validationExecution)); try { diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java index 0d2c9ac6..52b7f8f0 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java @@ -1,26 +1,11 @@ package io.dockstore.metricsaggregator.helper; -import static io.dockstore.common.metrics.FormatCheckHelper.checkExecutionDateISO8601Format; -import static java.util.stream.Collectors.groupingBy; - -import io.dockstore.metricsaggregator.DoubleStatistics; import io.dockstore.openapi.client.model.ExecutionStatusMetric; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; -import io.dockstore.openapi.client.model.ValidationExecution; import io.dockstore.openapi.client.model.ValidationStatusMetric; -import io.dockstore.openapi.client.model.ValidatorInfo; -import io.dockstore.openapi.client.model.ValidatorVersionInfo; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +35,7 @@ public static Optional getAggregatedMetrics(ExecutionsRequestBody allSu } // Set validation metrics - Optional aggregatedValidationStatus = getAggregatedValidationStatus(allSubmissions); + Optional aggregatedValidationStatus = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(allSubmissions); boolean containsValidationMetrics = aggregatedValidationStatus.isPresent(); aggregatedValidationStatus.ifPresent(aggregatedMetrics::setValidationStatus); @@ -63,173 +48,33 @@ public static Optional getAggregatedMetrics(ExecutionsRequestBody allSu } /** - * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of - * each validator tool version. - * @param allSubmissions - * @return - */ - public static Optional getAggregatedValidationStatus(ExecutionsRequestBody allSubmissions) { - // Get aggregated ValidationStatus metrics that were submitted to Dockstore - List validationStatusMetrics = allSubmissions.getAggregatedExecutions().stream() - .map(Metrics::getValidationStatus) - .filter(Objects::nonNull) - .collect(Collectors.toCollection(ArrayList::new)); - getAggregatedValidationStatusFromExecutions(allSubmissions.getValidationExecutions()).ifPresent(validationStatusMetrics::add); - - Map newValidatorToolToValidatorInfo = new HashMap<>(); - if (!validationStatusMetrics.isEmpty()) { - // Go through all the ValidationStatusMetrics and group the ValidationVersionInfos by validator tool - Map> validatorToolToValidationVersionInfos = validationStatusMetrics.stream() - .map(ValidationStatusMetric::getValidatorTools) - .flatMap(validatorToolToValidatorInfoMap -> validatorToolToValidatorInfoMap.entrySet().stream()) - .collect(groupingBy(Map.Entry::getKey, Collectors.flatMapping(entry -> { - ValidatorInfo validationInfoForValidatorTool = entry.getValue(); - return validationInfoForValidatorTool.getValidatorVersions().stream(); - }, Collectors.toList()))); - - // For each validator tool, find the most recent ValidatorVersionInfo for each version - validatorToolToValidationVersionInfos.forEach((validatorTool, validationVersionInfosByValidatorTool) -> { - // Number of runs across all versions - final int numberOfRuns = validationVersionInfosByValidatorTool.stream().map(ValidatorVersionInfo::getNumberOfRuns) - .mapToInt(Integer::intValue) - .sum(); - final List validationRunsStatistics = validationVersionInfosByValidatorTool.stream() - .map(validatorVersionInfo -> new DoubleStatistics(validatorVersionInfo.getPassingRate(), validatorVersionInfo.getNumberOfRuns())) - .toList(); - - final double passingRate = DoubleStatistics.createFromStatistics(validationRunsStatistics).getAverage(); - final Optional mostRecentValidationVersion = getLatestValidationVersionInfo(validationVersionInfosByValidatorTool); - - if (mostRecentValidationVersion.isPresent()) { - // Group ValidatorVersionInfo by version name - Map> versionNameToValidationVersionInfos = validationVersionInfosByValidatorTool.stream() - .collect(Collectors.groupingBy(ValidatorVersionInfo::getName)); - - // Get a list of the most recent ValidatorVersionInfo for each version - List mostRecentValidationVersionInfos = versionNameToValidationVersionInfos.values().stream().map(AggregationHelper::getLatestValidationVersionInfo).filter(Optional::isPresent).map(Optional::get).toList(); - - // Set validation info for the validator tool - ValidatorInfo validatorInfo = new ValidatorInfo() - .mostRecentVersionName(mostRecentValidationVersion.get().getName()) - .validatorVersions(mostRecentValidationVersionInfos) - .numberOfRuns(numberOfRuns) - .passingRate(passingRate); - - newValidatorToolToValidatorInfo.put(validatorTool, validatorInfo); - } - }); - } - - if (newValidatorToolToValidatorInfo.isEmpty()) { - return Optional.empty(); - } - - return Optional.of(new ValidationStatusMetric().validatorTools(newValidatorToolToValidatorInfo)); - } - - /** - * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of - * each validator tool version. - * @param executions + * Aggregates metrics into a single metric + * @param aggregatedMetrics * @return */ - public static Optional getAggregatedValidationStatusFromExecutions(List executions) { - if (executions.isEmpty()) { - return Optional.empty(); - } - - // Group executions by validator tool - Map> validatorToolToValidations = executions.stream() - .collect(groupingBy(ValidationExecution::getValidatorTool)); - - // For each validator tool, aggregate validation metrics for it - Map validatorToolToValidationInfo = new HashMap<>(); - validatorToolToValidations.forEach((validatorTool, validatorToolExecutions) -> { - Optional latestValidationExecution = getLatestValidationExecution(validatorToolExecutions); - - if (latestValidationExecution.isPresent()) { - // Group the validation executions for the validator tool by version - Map> validatorVersionNameToValidationExecutions = validatorToolExecutions.stream() - .collect(groupingBy(ValidationExecution::getValidatorToolVersion)); - - // Get the validation information for the most recent execution for each validator tool version - Map validatorVersionNameToVersionInfo = new HashMap<>(); - validatorVersionNameToValidationExecutions.forEach((validatorVersionName, validatorVersionExecutions) -> { - Optional latestValidationExecutionForVersion = getLatestValidationExecution(validatorVersionExecutions); - - latestValidationExecutionForVersion.ifPresent(validationExecution -> { - ValidatorVersionInfo validatorVersionInfo = new ValidatorVersionInfo() - .name(validatorVersionName) - .isValid(validationExecution.isIsValid()) - .dateExecuted(validationExecution.getDateExecuted()) - .numberOfRuns(validatorVersionExecutions.size()) - .passingRate(getPassingRate(validatorVersionExecutions)); - - if (!validationExecution.isIsValid() && StringUtils.isNotBlank(validationExecution.getErrorMessage())) { - validatorVersionInfo.errorMessage(validationExecution.getErrorMessage()); - } - - validatorVersionNameToVersionInfo.put(validatorVersionName, validatorVersionInfo); - }); - }); - - // Set validation info for the validator tool - ValidatorInfo validatorInfo = new ValidatorInfo() - .mostRecentVersionName(validatorVersionNameToVersionInfo.get(latestValidationExecution.get().getValidatorToolVersion()).getName()) - .validatorVersions(validatorVersionNameToVersionInfo.values().stream().toList()) - .numberOfRuns(validatorToolExecutions.size()) - .passingRate(getPassingRate(validatorToolExecutions)); - - validatorToolToValidationInfo.put(validatorTool.toString(), validatorInfo); - } - }); - - // This shouldn't happen because all validation executions should the required fields, but check anyway - if (validatorToolToValidationInfo.isEmpty()) { - return Optional.empty(); - } - return Optional.of(new ValidationStatusMetric().validatorTools(validatorToolToValidationInfo)); - } - - static Optional getLatestValidationExecution(List executions) { - if (executions.isEmpty()) { - return Optional.empty(); - } - - boolean containsInvalidDate = executions.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty()); - if (containsInvalidDate) { - return Optional.empty(); + public static Optional getAggregatedMetrics(List aggregatedMetrics) { + Metrics overallMetrics = new Metrics(); + // Set run metrics + Optional aggregatedExecutionStatus = new ExecutionStatusAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics); + boolean containsRunMetrics = aggregatedExecutionStatus.isPresent(); + if (aggregatedExecutionStatus.isPresent()) { + overallMetrics.setExecutionStatusCount(aggregatedExecutionStatus.get()); + new ExecutionTimeAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setExecutionTime); + new CpuAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setCpu); + new MemoryAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setMemory); + new CostAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setCost); } - return executions.stream() - .max(Comparator.comparing(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).get(), Date::compareTo)); - } - - static Optional getLatestValidationVersionInfo(List validationVersionInfos) { - if (validationVersionInfos.isEmpty()) { - return Optional.empty(); - } + // Set validation metrics + Optional aggregatedValidationStatus = new ValidationStatusAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics); + boolean containsValidationMetrics = aggregatedValidationStatus.isPresent(); + aggregatedValidationStatus.ifPresent(overallMetrics::setValidationStatus); - boolean containsInvalidDate = validationVersionInfos.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty()); - if (containsInvalidDate) { - return Optional.empty(); + // Only return aggregated metrics if it contains either run metrics or validation metrics + if (containsRunMetrics || containsValidationMetrics) { + return Optional.of(overallMetrics); } - return validationVersionInfos.stream() - .max(Comparator.comparing(validatorVersionInfo -> checkExecutionDateISO8601Format(validatorVersionInfo.getDateExecuted()).get(), Date::compareTo)); - } - - /** - * Gets the percentage of executions that passed validation - * @param executions - * @return - */ - @SuppressWarnings("checkstyle:magicnumber") - static double getPassingRate(List executions) { - final double numberOfPassingExecutions = executions.stream() - .filter(ValidationExecution::isIsValid) - .count(); - - return (numberOfPassingExecutions / executions.size()) * 100; + return Optional.empty(); } } diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java index 3b76a3be..f46207cf 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java @@ -6,6 +6,7 @@ import io.dockstore.metricsaggregator.MoneyStatistics; import io.dockstore.openapi.client.model.Cost; import io.dockstore.openapi.client.model.CostMetric; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; @@ -14,15 +15,20 @@ import java.util.Optional; import org.javamoney.moneta.Money; -public class CostAggregator implements RunExecutionAggregator { +public class CostAggregator implements ExecutionAggregator { @Override public CostMetric getMetricFromMetrics(Metrics metrics) { return metrics.getCost(); } @Override - public Cost getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getCost(); + public Cost getMetricFromExecution(RunExecution execution) { + return execution.getCost(); + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); } @Override @@ -46,8 +52,8 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - List submittedCosts = getNonNullMetricsFromRunExecutions(workflowExecutions); + public Optional getAggregatedMetricFromExecutions(List executions) { + List submittedCosts = getNonNullMetricsFromExecutions(executions); boolean containsMalformedCurrencies = submittedCosts.stream().anyMatch(cost -> !isValidCurrencyCode(cost.getCurrency())); // This shouldn't happen until we allow users to submit any currency they want diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java index 1254a788..8fdbb6c1 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java @@ -2,6 +2,7 @@ import io.dockstore.metricsaggregator.DoubleStatistics; import io.dockstore.openapi.client.model.CpuMetric; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; @@ -13,15 +14,20 @@ * Aggregate CPU metrics by calculating the minimum, maximum, and average. * @return */ -public class CpuAggregator implements RunExecutionAggregator { +public class CpuAggregator implements ExecutionAggregator { @Override public CpuMetric getMetricFromMetrics(Metrics metrics) { return metrics.getCpu(); } @Override - public Integer getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getCpuRequirements(); + public Integer getMetricFromExecution(RunExecution execution) { + return execution.getCpuRequirements(); + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); } @Override @@ -41,8 +47,8 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - List cpuRequirements = getNonNullMetricsFromRunExecutions(workflowExecutions).stream() + public Optional getAggregatedMetricFromExecutions(List executions) { + List cpuRequirements = getNonNullMetricsFromExecutions(executions).stream() .map(Integer::doubleValue) .toList(); if (!cpuRequirements.isEmpty()) { diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/RunExecutionAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java similarity index 72% rename from metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/RunExecutionAggregator.java rename to metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java index 2d592eb2..f732cd87 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/RunExecutionAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java @@ -2,7 +2,6 @@ import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; -import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; import java.util.ArrayList; import java.util.List; @@ -12,27 +11,29 @@ /** * An interface defining the methods needed to aggregate RunExecution's + * @param The type of execution, example: RunExecution or ValidationExecution * @param The aggregated metric from Metrics * @param The execution metric from RunExecution */ -public interface RunExecutionAggregator { +public interface ExecutionAggregator { M getMetricFromMetrics(Metrics metrics); - E getMetricFromRunExecution(RunExecution runExecution); + E getMetricFromExecution(T execution); + List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody); /** * Aggregates TaskExecutions that belong to a single workflow run into a workflow-level RunExecution * @param taskExecutionsForOneWorkflowRun * @return */ - Optional getWorkflowExecutionFromTaskExecutions(TaskExecutions taskExecutionsForOneWorkflowRun); + Optional getWorkflowExecutionFromTaskExecutions(TaskExecutions taskExecutionsForOneWorkflowRun); /** * Aggregates workflow executions into an aggregated metric. - * @param workflowExecutions + * @param executions * @return */ - Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions); + Optional getAggregatedMetricFromExecutions(List executions); /** * Aggregates a list of aggregated metrics into one aggregated metric. @@ -42,13 +43,13 @@ public interface RunExecutionAggregator { Optional getAggregatedMetricsFromAggregatedMetrics(List aggregatedMetrics); /** - * Returns a list of RunExecutions where the execution metric is not null. - * @param runExecutions + * Returns a list of executions of type T where the execution metric is not null. + * @param executions * @return */ - default List getNonNullMetricsFromRunExecutions(List runExecutions) { - return runExecutions.stream() - .map(this::getMetricFromRunExecution) + default List getNonNullMetricsFromExecutions(List executions) { + return executions.stream() + .map(this::getMetricFromExecution) .filter(Objects::nonNull) .toList(); } @@ -66,11 +67,11 @@ default List getNonNullMetricsFromRunExecutions(List runExecuti * @return */ default Optional getAggregatedMetricFromAllSubmissions(ExecutionsRequestBody allSubmissions) { - final List workflowExecutions = new ArrayList<>(allSubmissions.getRunExecutions()); + final List workflowExecutions = new ArrayList<>(getExecutionsFromExecutionRequestBody(allSubmissions)); // If task executions are present, calculate the workflow RunExecution containing the overall workflow-level execution time for each list of tasks if (!allSubmissions.getTaskExecutions().isEmpty()) { - final List calculatedWorkflowExecutionsFromTasks = allSubmissions.getTaskExecutions().stream() + final List calculatedWorkflowExecutionsFromTasks = allSubmissions.getTaskExecutions().stream() .map(this::getWorkflowExecutionFromTaskExecutions) .filter(Optional::isPresent) .map(Optional::get) @@ -85,7 +86,7 @@ default Optional getAggregatedMetricFromAllSubmissions(ExecutionsRequestBody .collect(Collectors.toCollection(ArrayList::new)); // Aggregate workflow executions into one metric and add it to the list of aggregated metrics - Optional aggregatedMetricFromWorkflowExecutions = getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + Optional aggregatedMetricFromWorkflowExecutions = getAggregatedMetricFromExecutions(workflowExecutions); aggregatedMetricFromWorkflowExecutions.ifPresent(aggregatedMetrics::add); if (!aggregatedMetrics.isEmpty()) { @@ -94,4 +95,11 @@ default Optional getAggregatedMetricFromAllSubmissions(ExecutionsRequestBody } return Optional.empty(); } + + default Optional getAggregatedMetricFromMetricsList(List metricsList) { + List specificMetrics = metricsList.stream() + .map(this::getMetricFromMetrics) + .toList(); + return getAggregatedMetricsFromAggregatedMetrics(specificMetrics); + } } diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregator.java index ec62f859..4aeac71b 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregator.java @@ -3,6 +3,7 @@ import static java.util.stream.Collectors.groupingBy; import io.dockstore.openapi.client.model.ExecutionStatusMetric; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum; @@ -19,7 +20,7 @@ /** * Aggregate Execution Status metrics by summing up the count of each Execution Status. */ -public class ExecutionStatusAggregator implements RunExecutionAggregator { +public class ExecutionStatusAggregator implements ExecutionAggregator { @Override public ExecutionStatusMetric getMetricFromMetrics(Metrics metrics) { @@ -27,8 +28,13 @@ public ExecutionStatusMetric getMetricFromMetrics(Metrics metrics) { } @Override - public ExecutionStatusEnum getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getExecutionStatus(); + public ExecutionStatusEnum getMetricFromExecution(RunExecution execution) { + return execution.getExecutionStatus(); + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); } @Override @@ -57,10 +63,10 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - if (!workflowExecutions.isEmpty()) { + public Optional getAggregatedMetricFromExecutions(List executions) { + if (!executions.isEmpty()) { // Calculate the status count from the workflow executions submitted - Map executionsStatusCount = workflowExecutions.stream() + Map executionsStatusCount = executions.stream() .map(execution -> execution.getExecutionStatus().toString()) .collect(groupingBy(Function.identity(), Collectors.reducing(0, e -> 1, Integer::sum))); return Optional.of(new ExecutionStatusMetric().count(executionsStatusCount)); diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregator.java index 0f05c4cd..fe44b607 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregator.java @@ -5,6 +5,7 @@ import io.dockstore.metricsaggregator.DoubleStatistics; import io.dockstore.openapi.client.model.ExecutionTimeMetric; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; @@ -19,10 +20,10 @@ /** * Aggregate Execution Time metrics by calculating the minimum, maximum, and average. */ -public final class ExecutionTimeAggregator implements RunExecutionAggregator { +public final class ExecutionTimeAggregator implements ExecutionAggregator { @Override - public String getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getExecutionTime(); + public String getMetricFromExecution(RunExecution execution) { + return execution.getExecutionTime(); } @Override @@ -30,6 +31,11 @@ public ExecutionTimeMetric getMetricFromMetrics(Metrics metrics) { return metrics.getExecutionTime(); } + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); + } + @Override public Optional getWorkflowExecutionFromTaskExecutions(TaskExecutions taskExecutionsForOneWorkflowRun) { final List taskExecutions = taskExecutionsForOneWorkflowRun.getTaskExecutions(); @@ -69,8 +75,8 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - List executionTimes = getNonNullMetricsFromRunExecutions(workflowExecutions); + public Optional getAggregatedMetricFromExecutions(List executions) { + List executionTimes = getNonNullMetricsFromExecutions(executions); boolean containsMalformedExecutionTimes = executionTimes.stream().anyMatch(executionTime -> checkExecutionTimeISO8601Format(executionTime).isEmpty()); // This really shouldn't happen because the webservice validates that the ExecutionTime is in the correct format diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/MemoryAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/MemoryAggregator.java index 07a34deb..25c16e1a 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/MemoryAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/MemoryAggregator.java @@ -1,6 +1,7 @@ package io.dockstore.metricsaggregator.helper; import io.dockstore.metricsaggregator.DoubleStatistics; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.MemoryMetric; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; @@ -12,15 +13,20 @@ /** * Aggregate Memory metrics by calculating the minimum, maximum, and average. */ -public class MemoryAggregator implements RunExecutionAggregator { +public class MemoryAggregator implements ExecutionAggregator { @Override public MemoryMetric getMetricFromMetrics(Metrics metrics) { return metrics.getMemory(); } @Override - public Double getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getMemoryRequirementsGB(); + public Double getMetricFromExecution(RunExecution execution) { + return execution.getMemoryRequirementsGB(); + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); } @Override @@ -40,8 +46,8 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - List memoryRequirements = getNonNullMetricsFromRunExecutions(workflowExecutions); + public Optional getAggregatedMetricFromExecutions(List executions) { + List memoryRequirements = getNonNullMetricsFromExecutions(executions); if (!memoryRequirements.isEmpty()) { DoubleStatistics statistics = new DoubleStatistics(memoryRequirements); return Optional.of(new MemoryMetric() diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java new file mode 100644 index 00000000..73c8ac73 --- /dev/null +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java @@ -0,0 +1,209 @@ +package io.dockstore.metricsaggregator.helper; + +import static io.dockstore.common.metrics.FormatCheckHelper.checkExecutionDateISO8601Format; +import static java.util.stream.Collectors.groupingBy; + +import io.dockstore.metricsaggregator.DoubleStatistics; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; +import io.dockstore.openapi.client.model.Metrics; +import io.dockstore.openapi.client.model.TaskExecutions; +import io.dockstore.openapi.client.model.ValidationExecution; +import io.dockstore.openapi.client.model.ValidationExecution.ValidatorToolEnum; +import io.dockstore.openapi.client.model.ValidationStatusMetric; +import io.dockstore.openapi.client.model.ValidatorInfo; +import io.dockstore.openapi.client.model.ValidatorVersionInfo; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; + +/** + * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of + * each validator tool version. + */ +public class ValidationStatusAggregator implements + ExecutionAggregator { + + @Override + public ValidationStatusMetric getMetricFromMetrics(Metrics metrics) { + return metrics.getValidationStatus(); + } + + @Override + public ValidationExecution getMetricFromExecution(ValidationExecution execution) { + return execution; // The entire execution contains the metric, not a specific field like with RunExecution + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getValidationExecutions(); + } + + /** + * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of + * each validator tool version. + * @param executions + * @return + */ + @Override + public Optional getAggregatedMetricFromExecutions(List executions) { + if (executions.isEmpty()) { + return Optional.empty(); + } + + // Group executions by validator tool + Map> validatorToolToValidations = executions.stream() + .collect(groupingBy(ValidationExecution::getValidatorTool)); + + // For each validator tool, aggregate validation metrics for it + Map validatorToolToValidationInfo = new HashMap<>(); + validatorToolToValidations.forEach((validatorTool, validatorToolExecutions) -> { + Optional latestValidationExecution = getLatestValidationExecution(validatorToolExecutions); + + if (latestValidationExecution.isPresent()) { + // Group the validation executions for the validator tool by version + Map> validatorVersionNameToValidationExecutions = validatorToolExecutions.stream() + .collect(groupingBy(ValidationExecution::getValidatorToolVersion)); + + // Get the validation information for the most recent execution for each validator tool version + Map validatorVersionNameToVersionInfo = new HashMap<>(); + validatorVersionNameToValidationExecutions.forEach((validatorVersionName, validatorVersionExecutions) -> { + Optional latestValidationExecutionForVersion = getLatestValidationExecution(validatorVersionExecutions); + + latestValidationExecutionForVersion.ifPresent(validationExecution -> { + ValidatorVersionInfo validatorVersionInfo = new ValidatorVersionInfo() + .name(validatorVersionName) + .isValid(validationExecution.isIsValid()) + .dateExecuted(validationExecution.getDateExecuted()) + .numberOfRuns(validatorVersionExecutions.size()) + .passingRate(getPassingRate(validatorVersionExecutions)); + + if (!validationExecution.isIsValid() && StringUtils.isNotBlank(validationExecution.getErrorMessage())) { + validatorVersionInfo.errorMessage(validationExecution.getErrorMessage()); + } + + validatorVersionNameToVersionInfo.put(validatorVersionName, validatorVersionInfo); + }); + }); + + // Set validation info for the validator tool + ValidatorInfo validatorInfo = new ValidatorInfo() + .mostRecentVersionName(validatorVersionNameToVersionInfo.get(latestValidationExecution.get().getValidatorToolVersion()).getName()) + .validatorVersions(validatorVersionNameToVersionInfo.values().stream().toList()) + .numberOfRuns(validatorToolExecutions.size()) + .passingRate(getPassingRate(validatorToolExecutions)); + + validatorToolToValidationInfo.put(validatorTool.toString(), validatorInfo); + } + }); + + // This shouldn't happen because all validation executions should the required fields, but check anyway + if (validatorToolToValidationInfo.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new ValidationStatusMetric().validatorTools(validatorToolToValidationInfo)); + } + + @Override + public Optional getWorkflowExecutionFromTaskExecutions(TaskExecutions taskExecutionsForOneWorkflowRun) { + return Optional.empty(); // Task executions don't apply to validation executions + } + + @Override + public Optional getAggregatedMetricsFromAggregatedMetrics(List aggregatedMetrics) { + Map newValidatorToolToValidatorInfo = new HashMap<>(); + if (!aggregatedMetrics.isEmpty()) { + // Go through all the ValidationStatusMetrics and group the ValidationVersionInfos by validator tool + Map> validatorToolToValidationVersionInfos = aggregatedMetrics.stream() + .map(ValidationStatusMetric::getValidatorTools) + .flatMap(validatorToolToValidatorInfoMap -> validatorToolToValidatorInfoMap.entrySet().stream()) + .collect(groupingBy(Map.Entry::getKey, Collectors.flatMapping(entry -> { + ValidatorInfo validationInfoForValidatorTool = entry.getValue(); + return validationInfoForValidatorTool.getValidatorVersions().stream(); + }, Collectors.toList()))); + + // For each validator tool, find the most recent ValidatorVersionInfo for each version + validatorToolToValidationVersionInfos.forEach((validatorTool, validationVersionInfosByValidatorTool) -> { + // Number of runs across all versions + final int numberOfRuns = validationVersionInfosByValidatorTool.stream().map(ValidatorVersionInfo::getNumberOfRuns) + .mapToInt(Integer::intValue) + .sum(); + final List validationRunsStatistics = validationVersionInfosByValidatorTool.stream() + .map(validatorVersionInfo -> new DoubleStatistics(validatorVersionInfo.getPassingRate(), validatorVersionInfo.getNumberOfRuns())) + .toList(); + + final double passingRate = DoubleStatistics.createFromStatistics(validationRunsStatistics).getAverage(); + final Optional mostRecentValidationVersion = getLatestValidationVersionInfo(validationVersionInfosByValidatorTool); + + if (mostRecentValidationVersion.isPresent()) { + // Group ValidatorVersionInfo by version name + Map> versionNameToValidationVersionInfos = validationVersionInfosByValidatorTool.stream() + .collect(Collectors.groupingBy(ValidatorVersionInfo::getName)); + + // Get a list of the most recent ValidatorVersionInfo for each version + List mostRecentValidationVersionInfos = versionNameToValidationVersionInfos.values().stream().map(this::getLatestValidationVersionInfo).filter(Optional::isPresent).map(Optional::get).toList(); + + // Set validation info for the validator tool + ValidatorInfo validatorInfo = new ValidatorInfo() + .mostRecentVersionName(mostRecentValidationVersion.get().getName()) + .validatorVersions(mostRecentValidationVersionInfos) + .numberOfRuns(numberOfRuns) + .passingRate(passingRate); + + newValidatorToolToValidatorInfo.put(validatorTool, validatorInfo); + } + }); + } + + if (newValidatorToolToValidatorInfo.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new ValidationStatusMetric().validatorTools(newValidatorToolToValidatorInfo)); + } + + static Optional getLatestValidationExecution(List executions) { + if (executions.isEmpty()) { + return Optional.empty(); + } + + boolean containsInvalidDate = executions.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty()); + if (containsInvalidDate) { + return Optional.empty(); + } + + return executions.stream() + .max(Comparator.comparing(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).get(), Date::compareTo)); + } + + Optional getLatestValidationVersionInfo(List validationVersionInfos) { + if (validationVersionInfos.isEmpty()) { + return Optional.empty(); + } + + boolean containsInvalidDate = validationVersionInfos.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty()); + if (containsInvalidDate) { + return Optional.empty(); + } + + return validationVersionInfos.stream() + .max(Comparator.comparing(validatorVersionInfo -> checkExecutionDateISO8601Format(validatorVersionInfo.getDateExecuted()).get(), Date::compareTo)); + } + + /** + * Gets the percentage of executions that passed validation + * @param executions + * @return + */ + @SuppressWarnings("checkstyle:magicnumber") + static double getPassingRate(List executions) { + final double numberOfPassingExecutions = executions.stream() + .filter(ValidationExecution::isIsValid) + .count(); + + return (numberOfPassingExecutions / executions.size()) * 100; + } +} diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java index 5b3ee632..3efe56c3 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java @@ -24,6 +24,7 @@ import static io.dockstore.metricsaggregator.common.TestUtilities.ENDPOINT_OVERRIDE; import static io.dockstore.metricsaggregator.common.TestUtilities.createRunExecution; import static io.dockstore.metricsaggregator.common.TestUtilities.createValidationExecution; +import static io.dockstore.metricsaggregator.common.TestUtilities.generateExecutionId; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_RUNTIME_INVALID; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_SEMANTIC_INVALID; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.SUCCESSFUL; @@ -66,6 +67,7 @@ import io.dropwizard.testing.DropwizardTestSupport; import io.dropwizard.testing.ResourceHelpers; import java.io.IOException; +import java.time.Instant; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -230,6 +232,8 @@ void testAggregateMetrics() { // Submit two TaskExecutions, each one representing the task metrics for a single workflow execution // A successful task execution that ran for 11 seconds, requires 6 CPUs and 5.5 GBs of memory. Signifies that this workflow execution only executed one task TaskExecutions taskExecutions = new TaskExecutions().taskExecutions(List.of(createRunExecution(SUCCESSFUL, "PT11S", 6, 5.5, new Cost().value(2.00), "us-central1"))); + taskExecutions.setDateExecuted(Instant.now().toString()); + taskExecutions.setExecutionId(generateExecutionId()); executionsRequestBody = new ExecutionsRequestBody().taskExecutions(List.of(taskExecutions)); // Submit metrics for the same workflow version for platform 1 extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform1, id, versionId, ""); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java index d79097a7..8714e7d8 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java @@ -27,6 +27,8 @@ import java.io.File; import java.time.Instant; import java.util.Optional; +import java.util.UUID; + import org.apache.commons.configuration2.INIConfiguration; public final class TestUtilities { @@ -46,15 +48,21 @@ public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum e .memoryRequirementsGB(memoryRequirementsGB) .cost(cost) .region(region); + runExecution.setExecutionId(generateExecutionId()); runExecution.setDateExecuted(Instant.now().toString()); return runExecution; } + public static String generateExecutionId() { + return UUID.randomUUID().toString(); + } + public static ValidationExecution createValidationExecution(ValidatorToolEnum validatorTool, String validatorToolVersion, boolean isValid) { ValidationExecution validationExecution = new ValidationExecution() .validatorTool(validatorTool) .validatorToolVersion(validatorToolVersion) .isValid(isValid); + validationExecution.setExecutionId(generateExecutionId()); validationExecution.setDateExecuted(Instant.now().toString()); return validationExecution; } diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java index 0f01f0f5..24f05965 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.dockstore.openapi.client.model.AggregatedExecution; import io.dockstore.openapi.client.model.Cost; import io.dockstore.openapi.client.model.CostMetric; import io.dockstore.openapi.client.model.CpuMetric; @@ -17,7 +18,6 @@ import io.dockstore.openapi.client.model.ExecutionTimeMetric; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.MemoryMetric; -import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; import io.dockstore.openapi.client.model.ValidationExecution; @@ -47,7 +47,8 @@ void testGetAggregatedExecutionStatus() { assertEquals(1, executionStatusMetric.get().getCount().get(SUCCESSFUL.toString())); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics().executionStatusCount( + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.executionStatusCount( new ExecutionStatusMetric().count( Map.of(SUCCESSFUL.toString(), 10, FAILED_RUNTIME_INVALID.toString(), 1))); allSubmissions = new ExecutionsRequestBody().runExecutions(List.of(submittedRunExecution)).aggregatedExecutions(List.of(submittedAggregatedMetrics)); @@ -122,12 +123,12 @@ void testGetAggregatedExecutionTime() { assertEquals(1, executionTimeMetric.get().getNumberOfDataPointsForAverage()); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics() - .executionTime(new ExecutionTimeMetric() - .minimum(2.0) - .maximum(6.0) - .average(4.0) - .numberOfDataPointsForAverage(2)); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.executionTime(new ExecutionTimeMetric() + .minimum(2.0) + .maximum(6.0) + .average(4.0) + .numberOfDataPointsForAverage(2)); executionTimeMetric = executionTimeAggregator.getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(executionTimeMetric.isPresent()); assertEquals(2.0, executionTimeMetric.get().getMinimum()); @@ -181,12 +182,12 @@ void testGetAggregatedCpu() { assertEquals(1, cpuMetric.get().getNumberOfDataPointsForAverage()); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics() - .cpu(new CpuMetric() - .minimum(2.0) - .maximum(6.0) - .average(4.0) - .numberOfDataPointsForAverage(2)); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.cpu(new CpuMetric() + .minimum(2.0) + .maximum(6.0) + .average(4.0) + .numberOfDataPointsForAverage(2)); cpuMetric = cpuAggregator.getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(cpuMetric.isPresent()); assertEquals(1.0, cpuMetric.get().getMinimum()); @@ -231,12 +232,12 @@ void testGetAggregatedMemory() { assertEquals(1, memoryMetric.get().getNumberOfDataPointsForAverage()); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics() - .memory(new MemoryMetric() - .minimum(2.0) - .maximum(6.0) - .average(4.0) - .numberOfDataPointsForAverage(2)); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.memory(new MemoryMetric() + .minimum(2.0) + .maximum(6.0) + .average(4.0) + .numberOfDataPointsForAverage(2)); memoryMetric = memoryAggregator.getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(memoryMetric.isPresent()); assertEquals(2.0, memoryMetric.get().getMinimum()); @@ -282,12 +283,12 @@ void testGetAggregatedCost() { assertEquals(1, costMetric.get().getNumberOfDataPointsForAverage()); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics() - .cost(new CostMetric() - .minimum(2.00) - .maximum(6.00) - .average(4.00) - .numberOfDataPointsForAverage(2)); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.cost(new CostMetric() + .minimum(2.00) + .maximum(6.00) + .average(4.00) + .numberOfDataPointsForAverage(2)); costMetric = costAggregator.getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(costMetric.isPresent()); assertEquals(2.0, costMetric.get().getMinimum()); @@ -313,14 +314,14 @@ void testGetAggregatedCost() { @Test void testGetAggregatedValidationStatus() { List executions = new ArrayList<>(); - Optional validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); + Optional validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isEmpty()); // Add an execution with validation data final ValidationExecution.ValidatorToolEnum validatorTool = ValidationExecution.ValidatorToolEnum.MINIWDL; final String validatorToolVersion1 = "1.0"; executions.add(createValidationExecution(validatorTool, validatorToolVersion1, true)); - validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); + validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); ValidatorInfo validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); assertNotNull(validatorInfo); @@ -338,7 +339,7 @@ void testGetAggregatedValidationStatus() { // Add an execution that isn't valid for the same validator final String validatorToolVersion2 = "2.0"; executions.add(createValidationExecution(validatorTool, validatorToolVersion2, false).errorMessage("This is an error message")); - validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); + validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); mostRecentValidatorVersion = validatorInfo.getValidatorVersions().stream().filter(validatorVersion -> validatorToolVersion2.equals(validatorVersion.getName())).findFirst().get(); @@ -356,7 +357,7 @@ void testGetAggregatedValidationStatus() { ValidationExecution validationExecution = createValidationExecution(validatorTool, validatorToolVersion1, true); validationExecution.setDateExecuted(expectedDateExecuted); executions.add(validationExecution); - validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); + validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); mostRecentValidatorVersion = validatorInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion1.equals(validationVersion.getName())).findFirst().get(); @@ -378,14 +379,14 @@ void testGetAggregatedValidationStatus() { .passingRate(100d) .numberOfRuns(4) .dateExecuted(expectedDateExecuted); - Metrics submittedAggregatedMetrics = new Metrics() - .validationStatus(new ValidationStatusMetric().validatorTools( - Map.of(validatorTool.toString(), new ValidatorInfo() - .validatorVersions(List.of(validationVersionInfo)) - .numberOfRuns(4) - .passingRate(100d)))); - - validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.validationStatus(new ValidationStatusMetric().validatorTools( + Map.of(validatorTool.toString(), new ValidatorInfo() + .validatorVersions(List.of(validationVersionInfo)) + .numberOfRuns(4) + .passingRate(100d)))); + + validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(validationStatusMetric.isPresent()); validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); mostRecentValidatorVersion = validatorInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion1.equals(validationVersion.getName())).findFirst().get(); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CostAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CostAggregatorTest.java index d3dc047c..6db0cd26 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CostAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CostAggregatorTest.java @@ -46,12 +46,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional costMetric = COST_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional costMetric = COST_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(costMetric.isEmpty()); // Test the metric calculated from a single workflow execution. The min, max, and average should be the same value as the single execution List workflowExecutions = List.of(new RunExecution().cost(new Cost().value(1.0))); - costMetric = COST_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + costMetric = COST_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(costMetric.isPresent()); assertEquals(1.0, costMetric.get().getMinimum()); assertEquals(1.0, costMetric.get().getMaximum()); @@ -64,7 +64,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().cost(new Cost().value(4.0)), new RunExecution().cost(new Cost().value(6.0)) ); - costMetric = COST_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + costMetric = COST_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(costMetric.isPresent()); assertEquals(2.0, costMetric.get().getMinimum()); assertEquals(6.0, costMetric.get().getMaximum()); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CpuAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CpuAggregatorTest.java index f551a52e..d4cbb28d 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CpuAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CpuAggregatorTest.java @@ -45,12 +45,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(cpuMetric.isEmpty()); // Test the metric calculated from a single workflow execution. The min, max, and average should be the same value as the single execution List workflowExecutions = List.of(new RunExecution().cpuRequirements(1)); - cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(cpuMetric.isPresent()); assertEquals(1.0, cpuMetric.get().getMinimum()); assertEquals(1.0, cpuMetric.get().getMaximum()); @@ -63,7 +63,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().cpuRequirements(4), new RunExecution().cpuRequirements(6) ); - cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(cpuMetric.isPresent()); assertEquals(2.0, cpuMetric.get().getMinimum()); assertEquals(6.0, cpuMetric.get().getMaximum()); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregatorTest.java index f6f3695c..6f6971ea 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregatorTest.java @@ -71,12 +71,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(executionStatusMetric.isEmpty()); // Test the metric calculated from a single workflow execution. List workflowExecutions = List.of(new RunExecution().executionStatus(SUCCESSFUL)); - executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(executionStatusMetric.isPresent()); assertEquals(1, executionStatusMetric.get().getCount().get(SUCCESSFUL.toString())); assertFalse(executionStatusMetric.get().getCount().containsKey(FAILED_SEMANTIC_INVALID.toString())); @@ -88,7 +88,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().executionStatus(FAILED_SEMANTIC_INVALID), new RunExecution().executionStatus(FAILED_RUNTIME_INVALID) ); - executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(executionStatusMetric.isPresent()); assertEquals(1, executionStatusMetric.get().getCount().get(SUCCESSFUL.toString())); assertEquals(1, executionStatusMetric.get().getCount().get(FAILED_SEMANTIC_INVALID.toString())); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregatorTest.java index 9e799247..ad369134 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregatorTest.java @@ -53,12 +53,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(executionTimeMetric.isEmpty()); // Test the metric calculated from a single workflow execution. The min, max, and average should be the same value as the single execution List workflowExecutions = List.of(new RunExecution().executionTime("PT10S")); // 10 seconds - executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(executionTimeMetric.isPresent()); assertEquals(10.0, executionTimeMetric.get().getMinimum()); assertEquals(10.0, executionTimeMetric.get().getMaximum()); @@ -71,7 +71,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().executionTime("PT20S"), // 20 seconds new RunExecution().executionTime("PT30S") // 30 seconds ); - executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(executionTimeMetric.isPresent()); assertEquals(10.0, executionTimeMetric.get().getMinimum()); assertEquals(30.0, executionTimeMetric.get().getMaximum()); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/MemoryAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/MemoryAggregatorTest.java index a897ae11..77da3552 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/MemoryAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/MemoryAggregatorTest.java @@ -45,12 +45,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(memoryMetric.isEmpty()); // Test the memory metric calculated from a single workflow execution. The min, max, and average should be the same value as the single execution List workflowExecutions = List.of(new RunExecution().memoryRequirementsGB(1.0)); - memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(memoryMetric.isPresent()); assertEquals(1.0, memoryMetric.get().getMinimum()); assertEquals(1.0, memoryMetric.get().getMaximum()); @@ -63,7 +63,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().memoryRequirementsGB(4.0), new RunExecution().memoryRequirementsGB(6.0) ); - memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(memoryMetric.isPresent()); assertEquals(2.0, memoryMetric.get().getMinimum()); assertEquals(6.0, memoryMetric.get().getMaximum());