Skip to content

Commit

Permalink
Changes required for changes in webservice
Browse files Browse the repository at this point in the history
  • Loading branch information
@kathy-t
kathy-t committed Jan 7, 2024
1 parent 2936d16 commit d48070e
Show file tree
Hide file tree
Showing 18 changed files with 381 additions and 272 deletions.
5 changes: 3 additions & 2 deletions ...icsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import io.dockstore.common.metrics.MetricsData;
import io.dockstore.common.metrics.MetricsDataS3Client;
import io.dockstore.openapi.client.api.ExtendedGa4GhApi;
import io.dockstore.openapi.client.model.AggregatedExecution;
import io.dockstore.openapi.client.model.ExecutionsRequestBody;
import io.dockstore.openapi.client.model.Metrics;
import io.dockstore.openapi.client.model.RunExecution;
Expand Down Expand Up @@ -108,7 +109,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) {
if (!allMetrics.isEmpty()) {
// Calculate metrics across all platforms by aggregating the aggregated metrics from each platform
try {
getAggregatedMetrics(new ExecutionsRequestBody().aggregatedExecutions(allMetrics)).ifPresent(metrics -> {
getAggregatedMetrics(allMetrics).ifPresent(metrics -> {
extendedGa4GhApi.aggregatedMetricsPut(metrics, Partner.ALL.name(), toolId, versionName);
System.out.printf("Aggregated metrics across all platforms (%s) for tool ID %s, version %s from directory %s%n",
platformsString, toolId, versionName, versionS3KeyPrefix);
Expand All @@ -135,7 +136,7 @@ private ExecutionsRequestBody getExecutions(String toolId, String versionName, S
List<RunExecution> runExecutionsFromAllSubmissions = new ArrayList<>();
List<TaskExecutions> taskExecutionsFromAllSubmissions = new ArrayList<>();
List<ValidationExecution> validationExecutionsFromAllSubmissions = new ArrayList<>();
List<Metrics> aggregatedExecutionsFromAllSubmissions = new ArrayList<>();
List<AggregatedExecution> aggregatedExecutionsFromAllSubmissions = new ArrayList<>();

for (MetricsData metricsData : metricsDataList) {
String fileContent = metricsDataS3Client.getMetricsDataFileContent(metricsData.toolId(), metricsData.toolVersionName(),
Expand Down
3 changes: 3 additions & 0 deletions ...ator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
import java.nio.file.Files;
import java.util.List;
import java.util.Optional;
import java.util.UUID;

import org.apache.commons.configuration2.INIConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -184,6 +186,7 @@ private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolE
.validatorToolVersion(validatorVersion)
.isValid(isValid);
validationExecution.setDateExecuted(dateExecuted);
validationExecution.setExecutionId(UUID.randomUUID().toString()); // No execution ID was provided by DNAstack, generate a random one
ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().validationExecutions(List.of(validationExecution));

try {
Expand Down
199 changes: 22 additions & 177 deletions metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,11 @@
package io.dockstore.metricsaggregator.helper;

import static io.dockstore.common.metrics.FormatCheckHelper.checkExecutionDateISO8601Format;
import static java.util.stream.Collectors.groupingBy;

import io.dockstore.metricsaggregator.DoubleStatistics;
import io.dockstore.openapi.client.model.ExecutionStatusMetric;
import io.dockstore.openapi.client.model.ExecutionsRequestBody;
import io.dockstore.openapi.client.model.Metrics;
import io.dockstore.openapi.client.model.ValidationExecution;
import io.dockstore.openapi.client.model.ValidationStatusMetric;
import io.dockstore.openapi.client.model.ValidatorInfo;
import io.dockstore.openapi.client.model.ValidatorVersionInfo;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -50,7 +35,7 @@ public static Optional<Metrics> getAggregatedMetrics(ExecutionsRequestBody allSu
}

// Set validation metrics
Optional<ValidationStatusMetric> aggregatedValidationStatus = getAggregatedValidationStatus(allSubmissions);
Optional<ValidationStatusMetric> aggregatedValidationStatus = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(allSubmissions);
boolean containsValidationMetrics = aggregatedValidationStatus.isPresent();
aggregatedValidationStatus.ifPresent(aggregatedMetrics::setValidationStatus);

Expand All @@ -63,173 +48,33 @@ public static Optional<Metrics> getAggregatedMetrics(ExecutionsRequestBody allSu
}

/**
* Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of
* each validator tool version.
* @param allSubmissions
* @return
*/
public static Optional<ValidationStatusMetric> getAggregatedValidationStatus(ExecutionsRequestBody allSubmissions) {
// Get aggregated ValidationStatus metrics that were submitted to Dockstore
List<ValidationStatusMetric> validationStatusMetrics = allSubmissions.getAggregatedExecutions().stream()
.map(Metrics::getValidationStatus)
.filter(Objects::nonNull)
.collect(Collectors.toCollection(ArrayList::new));
getAggregatedValidationStatusFromExecutions(allSubmissions.getValidationExecutions()).ifPresent(validationStatusMetrics::add);

Map<String, ValidatorInfo> newValidatorToolToValidatorInfo = new HashMap<>();
if (!validationStatusMetrics.isEmpty()) {
// Go through all the ValidationStatusMetrics and group the ValidationVersionInfos by validator tool
Map<String, List<ValidatorVersionInfo>> validatorToolToValidationVersionInfos = validationStatusMetrics.stream()
.map(ValidationStatusMetric::getValidatorTools)
.flatMap(validatorToolToValidatorInfoMap -> validatorToolToValidatorInfoMap.entrySet().stream())
.collect(groupingBy(Map.Entry::getKey, Collectors.flatMapping(entry -> {
ValidatorInfo validationInfoForValidatorTool = entry.getValue();
return validationInfoForValidatorTool.getValidatorVersions().stream();
}, Collectors.toList())));

// For each validator tool, find the most recent ValidatorVersionInfo for each version
validatorToolToValidationVersionInfos.forEach((validatorTool, validationVersionInfosByValidatorTool) -> {
// Number of runs across all versions
final int numberOfRuns = validationVersionInfosByValidatorTool.stream().map(ValidatorVersionInfo::getNumberOfRuns)
.mapToInt(Integer::intValue)
.sum();
final List<DoubleStatistics> validationRunsStatistics = validationVersionInfosByValidatorTool.stream()
.map(validatorVersionInfo -> new DoubleStatistics(validatorVersionInfo.getPassingRate(), validatorVersionInfo.getNumberOfRuns()))
.toList();

final double passingRate = DoubleStatistics.createFromStatistics(validationRunsStatistics).getAverage();
final Optional<ValidatorVersionInfo> mostRecentValidationVersion = getLatestValidationVersionInfo(validationVersionInfosByValidatorTool);

if (mostRecentValidationVersion.isPresent()) {
// Group ValidatorVersionInfo by version name
Map<String, List<ValidatorVersionInfo>> versionNameToValidationVersionInfos = validationVersionInfosByValidatorTool.stream()
.collect(Collectors.groupingBy(ValidatorVersionInfo::getName));

// Get a list of the most recent ValidatorVersionInfo for each version
List<ValidatorVersionInfo> mostRecentValidationVersionInfos = versionNameToValidationVersionInfos.values().stream().map(AggregationHelper::getLatestValidationVersionInfo).filter(Optional::isPresent).map(Optional::get).toList();

// Set validation info for the validator tool
ValidatorInfo validatorInfo = new ValidatorInfo()
.mostRecentVersionName(mostRecentValidationVersion.get().getName())
.validatorVersions(mostRecentValidationVersionInfos)
.numberOfRuns(numberOfRuns)
.passingRate(passingRate);

newValidatorToolToValidatorInfo.put(validatorTool, validatorInfo);
}
});
}

if (newValidatorToolToValidatorInfo.isEmpty()) {
return Optional.empty();
}

return Optional.of(new ValidationStatusMetric().validatorTools(newValidatorToolToValidatorInfo));
}

/**
* Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of
* each validator tool version.
* @param executions
* Aggregates metrics into a single metric
* @param aggregatedMetrics
* @return
*/
public static Optional<ValidationStatusMetric> getAggregatedValidationStatusFromExecutions(List<ValidationExecution> executions) {
if (executions.isEmpty()) {
return Optional.empty();
}

// Group executions by validator tool
Map<ValidationExecution.ValidatorToolEnum, List<ValidationExecution>> validatorToolToValidations = executions.stream()
.collect(groupingBy(ValidationExecution::getValidatorTool));

// For each validator tool, aggregate validation metrics for it
Map<String, ValidatorInfo> validatorToolToValidationInfo = new HashMap<>();
validatorToolToValidations.forEach((validatorTool, validatorToolExecutions) -> {
Optional<ValidationExecution> latestValidationExecution = getLatestValidationExecution(validatorToolExecutions);

if (latestValidationExecution.isPresent()) {
// Group the validation executions for the validator tool by version
Map<String, List<ValidationExecution>> validatorVersionNameToValidationExecutions = validatorToolExecutions.stream()
.collect(groupingBy(ValidationExecution::getValidatorToolVersion));

// Get the validation information for the most recent execution for each validator tool version
Map<String, ValidatorVersionInfo> validatorVersionNameToVersionInfo = new HashMap<>();
validatorVersionNameToValidationExecutions.forEach((validatorVersionName, validatorVersionExecutions) -> {
Optional<ValidationExecution> latestValidationExecutionForVersion = getLatestValidationExecution(validatorVersionExecutions);

latestValidationExecutionForVersion.ifPresent(validationExecution -> {
ValidatorVersionInfo validatorVersionInfo = new ValidatorVersionInfo()
.name(validatorVersionName)
.isValid(validationExecution.isIsValid())
.dateExecuted(validationExecution.getDateExecuted())
.numberOfRuns(validatorVersionExecutions.size())
.passingRate(getPassingRate(validatorVersionExecutions));

if (!validationExecution.isIsValid() && StringUtils.isNotBlank(validationExecution.getErrorMessage())) {
validatorVersionInfo.errorMessage(validationExecution.getErrorMessage());
}

validatorVersionNameToVersionInfo.put(validatorVersionName, validatorVersionInfo);
});
});

// Set validation info for the validator tool
ValidatorInfo validatorInfo = new ValidatorInfo()
.mostRecentVersionName(validatorVersionNameToVersionInfo.get(latestValidationExecution.get().getValidatorToolVersion()).getName())
.validatorVersions(validatorVersionNameToVersionInfo.values().stream().toList())
.numberOfRuns(validatorToolExecutions.size())
.passingRate(getPassingRate(validatorToolExecutions));

validatorToolToValidationInfo.put(validatorTool.toString(), validatorInfo);
}
});

// This shouldn't happen because all validation executions should the required fields, but check anyway
if (validatorToolToValidationInfo.isEmpty()) {
return Optional.empty();
}
return Optional.of(new ValidationStatusMetric().validatorTools(validatorToolToValidationInfo));
}

static Optional<ValidationExecution> getLatestValidationExecution(List<ValidationExecution> executions) {
if (executions.isEmpty()) {
return Optional.empty();
}

boolean containsInvalidDate = executions.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty());
if (containsInvalidDate) {
return Optional.empty();
public static Optional<Metrics> getAggregatedMetrics(List<Metrics> aggregatedMetrics) {
Metrics overallMetrics = new Metrics();
// Set run metrics
Optional<ExecutionStatusMetric> aggregatedExecutionStatus = new ExecutionStatusAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics);
boolean containsRunMetrics = aggregatedExecutionStatus.isPresent();
if (aggregatedExecutionStatus.isPresent()) {
overallMetrics.setExecutionStatusCount(aggregatedExecutionStatus.get());
new ExecutionTimeAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setExecutionTime);
new CpuAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setCpu);
new MemoryAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setMemory);
new CostAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setCost);
}

return executions.stream()
.max(Comparator.comparing(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).get(), Date::compareTo));
}

static Optional<ValidatorVersionInfo> getLatestValidationVersionInfo(List<ValidatorVersionInfo> validationVersionInfos) {
if (validationVersionInfos.isEmpty()) {
return Optional.empty();
}
// Set validation metrics
Optional<ValidationStatusMetric> aggregatedValidationStatus = new ValidationStatusAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics);
boolean containsValidationMetrics = aggregatedValidationStatus.isPresent();
aggregatedValidationStatus.ifPresent(overallMetrics::setValidationStatus);

boolean containsInvalidDate = validationVersionInfos.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty());
if (containsInvalidDate) {
return Optional.empty();
// Only return aggregated metrics if it contains either run metrics or validation metrics
if (containsRunMetrics || containsValidationMetrics) {
return Optional.of(overallMetrics);
}

return validationVersionInfos.stream()
.max(Comparator.comparing(validatorVersionInfo -> checkExecutionDateISO8601Format(validatorVersionInfo.getDateExecuted()).get(), Date::compareTo));
}

/**
* Gets the percentage of executions that passed validation
* @param executions
* @return
*/
@SuppressWarnings("checkstyle:magicnumber")
static double getPassingRate(List<ValidationExecution> executions) {
final double numberOfPassingExecutions = executions.stream()
.filter(ValidationExecution::isIsValid)
.count();

return (numberOfPassingExecutions / executions.size()) * 100;
return Optional.empty();
}
}
16 changes: 11 additions & 5 deletions metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import io.dockstore.metricsaggregator.MoneyStatistics;
import io.dockstore.openapi.client.model.Cost;
import io.dockstore.openapi.client.model.CostMetric;
import io.dockstore.openapi.client.model.ExecutionsRequestBody;
import io.dockstore.openapi.client.model.Metrics;
import io.dockstore.openapi.client.model.RunExecution;
import io.dockstore.openapi.client.model.TaskExecutions;
Expand All @@ -14,15 +15,20 @@
import java.util.Optional;
import org.javamoney.moneta.Money;

public class CostAggregator implements RunExecutionAggregator<CostMetric, Cost> {
public class CostAggregator implements ExecutionAggregator<RunExecution, CostMetric, Cost> {
@Override
public CostMetric getMetricFromMetrics(Metrics metrics) {
return metrics.getCost();
}

@Override
public Cost getMetricFromRunExecution(RunExecution runExecution) {
return runExecution.getCost();
public Cost getMetricFromExecution(RunExecution execution) {
return execution.getCost();
}

@Override
public List<RunExecution> getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) {
return executionsRequestBody.getRunExecutions();
}

@Override
Expand All @@ -46,8 +52,8 @@ public Optional<RunExecution> getWorkflowExecutionFromTaskExecutions(TaskExecuti
}

@Override
public Optional<CostMetric> getAggregatedMetricFromWorkflowExecutions(List<RunExecution> workflowExecutions) {
List<Cost> submittedCosts = getNonNullMetricsFromRunExecutions(workflowExecutions);
public Optional<CostMetric> getAggregatedMetricFromExecutions(List<RunExecution> executions) {
List<Cost> submittedCosts = getNonNullMetricsFromExecutions(executions);

boolean containsMalformedCurrencies = submittedCosts.stream().anyMatch(cost -> !isValidCurrencyCode(cost.getCurrency()));
// This shouldn't happen until we allow users to submit any currency they want
Expand Down
16 changes: 11 additions & 5 deletions metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.dockstore.metricsaggregator.DoubleStatistics;
import io.dockstore.openapi.client.model.CpuMetric;
import io.dockstore.openapi.client.model.ExecutionsRequestBody;
import io.dockstore.openapi.client.model.Metrics;
import io.dockstore.openapi.client.model.RunExecution;
import io.dockstore.openapi.client.model.TaskExecutions;
Expand All @@ -13,15 +14,20 @@
* Aggregate CPU metrics by calculating the minimum, maximum, and average.
* @return
*/
public class CpuAggregator implements RunExecutionAggregator<CpuMetric, Integer> {
public class CpuAggregator implements ExecutionAggregator<RunExecution, CpuMetric, Integer> {
@Override
public CpuMetric getMetricFromMetrics(Metrics metrics) {
return metrics.getCpu();
}

@Override
public Integer getMetricFromRunExecution(RunExecution runExecution) {
return runExecution.getCpuRequirements();
public Integer getMetricFromExecution(RunExecution execution) {
return execution.getCpuRequirements();
}

@Override
public List<RunExecution> getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) {
return executionsRequestBody.getRunExecutions();
}

@Override
Expand All @@ -41,8 +47,8 @@ public Optional<RunExecution> getWorkflowExecutionFromTaskExecutions(TaskExecuti
}

@Override
public Optional<CpuMetric> getAggregatedMetricFromWorkflowExecutions(List<RunExecution> workflowExecutions) {
List<Double> cpuRequirements = getNonNullMetricsFromRunExecutions(workflowExecutions).stream()
public Optional<CpuMetric> getAggregatedMetricFromExecutions(List<RunExecution> executions) {
List<Double> cpuRequirements = getNonNullMetricsFromExecutions(executions).stream()
.map(Integer::doubleValue)
.toList();
if (!cpuRequirements.isEmpty()) {
Expand Down
Loading

0 comments on commit d48070e

Please sign in to comment.