Skip to content

Commit

Permalink
Aggregate cost
Browse files Browse the repository at this point in the history
  • Loading branch information
@kathy-t
kathy-t committed Jun 28, 2023
1 parent cdb0c1c commit 379117d
Show file tree
Hide file tree
Showing 7 changed files with 157 additions and 81 deletions.
2 changes: 2 additions & 0 deletions ...icsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) {
return;
}

System.out.println("Aggregating metrics...");
for (S3DirectoryInfo directoryInfo : metricsDirectories) {
String toolId = directoryInfo.toolId();
String versionName = directoryInfo.versionId();
Expand Down Expand Up @@ -118,6 +119,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) {
}
}
}
System.out.println("Completed aggregating metrics");
}

/**
Expand Down
49 changes: 49 additions & 0 deletions metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import static java.util.stream.Collectors.groupingBy;

import io.dockstore.metricsaggregator.Statistics;
import io.dockstore.openapi.client.model.CostMetric;
import io.dockstore.openapi.client.model.CpuMetric;
import io.dockstore.openapi.client.model.ExecutionStatusMetric;
import io.dockstore.openapi.client.model.ExecutionTimeMetric;
Expand Down Expand Up @@ -54,6 +55,7 @@ public static Optional<Metrics> getAggregatedMetrics(ExecutionsRequestBody allSu
getAggregatedExecutionTime(allSubmissions).ifPresent(aggregatedMetrics::setExecutionTime);
getAggregatedCpu(allSubmissions).ifPresent(aggregatedMetrics::setCpu);
getAggregatedMemory(allSubmissions).ifPresent(aggregatedMetrics::setMemory);
getAggregatedCost(allSubmissions).ifPresent(aggregatedMetrics::setCost);
}

// Set validation metrics
Expand Down Expand Up @@ -259,6 +261,53 @@ public static Optional<MemoryMetric> getAggregatedMemoryFromExecutions(List<RunE
return Optional.empty();
}

/**
* Aggregate Cost metrics from all submissions by calculating the minimum, maximum, and average.
* @param allSubmissions
* @return
*/
public static Optional<CostMetric> getAggregatedCost(ExecutionsRequestBody allSubmissions) {
// Get aggregated Execution Time metrics that were submitted to Dockstore
List<CostMetric> costMetrics = allSubmissions.getAggregatedExecutions().stream()
.map(Metrics::getCost)
.filter(Objects::nonNull)
.collect(Collectors.toList());
getAggregatedCostFromExecutions(allSubmissions.getRunExecutions()).ifPresent(costMetrics::add);

if (!costMetrics.isEmpty()) {
List<Statistics> statistics = costMetrics.stream()
.map(metric -> new Statistics(metric.getMinimum(), metric.getMaximum(), metric.getAverage(), metric.getNumberOfDataPointsForAverage())).toList();
Statistics newStatistic = Statistics.createFromStatistics(statistics);
return Optional.of(new CostMetric()
.minimum(newStatistic.min())
.maximum(newStatistic.max())
.average(newStatistic.average())
.numberOfDataPointsForAverage(newStatistic.numberOfDataPoints()));
}
return Optional.empty();
}

/**
* Aggregate Cost metrics from the list of run executions by calculating the minimum, maximum, and average.
* @param executions
* @return
*/
public static Optional<CostMetric> getAggregatedCostFromExecutions(List<RunExecution> executions) {
List<Double> costs = executions.stream()
.map(RunExecution::getCostUSD)
.filter(Objects::nonNull)
.toList();
if (!costs.isEmpty()) {
Statistics statistics = new Statistics(costs);
return Optional.of(new CostMetric()
.minimum(statistics.min())
.maximum(statistics.max())
.average(statistics.average())
.numberOfDataPointsForAverage(statistics.numberOfDataPoints()));
}
return Optional.empty();
}

/**
* Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of
* each validator tool version.
Expand Down
2 changes: 1 addition & 1 deletion ...saggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ void testGetDirectories() {
final String workflowVersionId = version.getName();

// A successful execution that ran for 5 minutes, requires 2 CPUs and 2 GBs of memory
RunExecution execution = createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0);
RunExecution execution = createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0, 2.00, "us-central1");
ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().runExecutions(List.of(execution));
extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform1, workflowId, workflowVersionId, "");
extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform2, workflowId, workflowVersionId, "");
Expand Down
139 changes: 62 additions & 77 deletions ...or/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ void testAggregateMetrics() {
String versionId = version.getName();

// A successful run execution that ran for 5 minutes, requires 2 CPUs and 2 GBs of memory
List<RunExecution> runExecutions = List.of(createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0));
List<RunExecution> runExecutions = List.of(createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0, 2.00, "us-central1"));
// A successful miniwdl validation
final String validatorToolVersion1 = "1.0";
ValidationExecution validationExecution1 = new ValidationExecution()
Expand All @@ -164,15 +164,17 @@ void testAggregateMetrics() {
version = workflow.getWorkflowVersions().stream().filter(v -> "master".equals(v.getName())).findFirst().orElse(null);
assertNotNull(version);
assertEquals(expectedNumberOfPlatforms, version.getMetricsByPlatform().size(), "There should be metrics for two platforms");
assertAggregatedMetricsForPlatform(platform1, version, validationExecution1);
assertAggregatedMetricsForPlatform(platform2, version, validationExecution2);

Metrics platform1Metrics = version.getMetricsByPlatform().get(platform1);
assertNotNull(platform1Metrics);

compareAggregateMetricsWithPlatforms(platform2, version, validatorToolVersion1, validatorToolVersion2, platform1Metrics);
ValidatorVersionInfo mostRecentValidationVersionInfo;
ValidatorInfo validationInfo;

// A failed run execution that ran for 1 second, requires 2 CPUs and 4.5 GBs of memory
runExecutions = List.of(createRunExecution(FAILED_RUNTIME_INVALID, "PT1S", 4, 4.5));
runExecutions = List.of(createRunExecution(FAILED_RUNTIME_INVALID, "PT1S", 4, 4.5, 2.00, "us-central1"));
// A failed miniwdl validation for the same validator version
List<ValidationExecution> validationExecutions = List.of(new ValidationExecution().validatorTool(MINIWDL).validatorToolVersion("1.0").isValid(false).dateExecuted(Instant.now().toString()));
ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().runExecutions(runExecutions).validationExecutions(validationExecutions);
Expand Down Expand Up @@ -206,6 +208,12 @@ void testAggregateMetrics() {
assertEquals(3.25, platform1Metrics.getMemory().getAverage());
assertNotNull(platform1Metrics.getMemory().getUnit());

assertEquals(2, platform1Metrics.getCost().getNumberOfDataPointsForAverage());
assertEquals(2, platform1Metrics.getCost().getMinimum());
assertEquals(2, platform1Metrics.getCost().getMaximum());
assertEquals(2, platform1Metrics.getCost().getAverage());
assertNotNull(platform1Metrics.getCost().getUnit());

assertEquals(2, platform1Metrics.getExecutionTime().getNumberOfDataPointsForAverage());
assertEquals(1, platform1Metrics.getExecutionTime().getMinimum());
assertEquals(300, platform1Metrics.getExecutionTime().getMaximum());
Expand All @@ -224,89 +232,66 @@ void testAggregateMetrics() {
assertEquals(50d, validationInfo.getPassingRate());
assertEquals(2, validationInfo.getNumberOfRuns());

testAggregatedMetrics(version, validatorToolVersion1, validatorToolVersion2, platform1Metrics);
testOverallAggregatedMetrics(version, validatorToolVersion1, validatorToolVersion2, platform1Metrics);
}

private static void compareAggregateMetricsWithPlatforms(String platform2, WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) {
// Verify that the aggregated metrics are the same as the single execution for platform1
assertEquals(1, platform1Metrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions());
assertEquals(0, platform1Metrics.getExecutionStatusCount().getNumberOfFailedExecutions());
assertEquals(1, platform1Metrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name()));
assertFalse(platform1Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name()));
assertFalse(platform1Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name()));

assertEquals(1, platform1Metrics.getCpu().getNumberOfDataPointsForAverage());
assertEquals(2, platform1Metrics.getCpu().getMinimum());
assertEquals(2, platform1Metrics.getCpu().getMaximum());
assertEquals(2, platform1Metrics.getCpu().getAverage());
assertNull(platform1Metrics.getCpu().getUnit());

assertEquals(1, platform1Metrics.getMemory().getNumberOfDataPointsForAverage());
assertEquals(2, platform1Metrics.getMemory().getMinimum());
assertEquals(2, platform1Metrics.getMemory().getMaximum());
assertEquals(2, platform1Metrics.getMemory().getAverage());
assertNotNull(platform1Metrics.getMemory().getUnit());

assertEquals(1, platform1Metrics.getExecutionTime().getNumberOfDataPointsForAverage());
assertEquals(300, platform1Metrics.getExecutionTime().getMinimum());
assertEquals(300, platform1Metrics.getExecutionTime().getMaximum());
assertEquals(300, platform1Metrics.getExecutionTime().getAverage());
assertNotNull(platform1Metrics.getExecutionTime().getUnit());

assertEquals(1, platform1Metrics.getValidationStatus().getValidatorTools().size());
ValidatorInfo validationInfo = platform1Metrics.getValidationStatus().getValidatorTools().get(MINIWDL.toString());
private static void assertAggregatedMetricsForPlatform(String platform, WorkflowVersion version, ValidationExecution submittedValidationExecution) {
Metrics platformMetrics = version.getMetricsByPlatform().get(platform);
assertNotNull(platformMetrics);

// Verify that the aggregated metrics are the same as the single execution for the platform
assertEquals(1, platformMetrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions());
assertEquals(0, platformMetrics.getExecutionStatusCount().getNumberOfFailedExecutions());
assertEquals(1, platformMetrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name()));
assertFalse(platformMetrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name()));
assertFalse(platformMetrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name()));

assertEquals(1, platformMetrics.getCpu().getNumberOfDataPointsForAverage());
assertEquals(2, platformMetrics.getCpu().getMinimum());
assertEquals(2, platformMetrics.getCpu().getMaximum());
assertEquals(2, platformMetrics.getCpu().getAverage());
assertNull(platformMetrics.getCpu().getUnit());

assertEquals(1, platformMetrics.getMemory().getNumberOfDataPointsForAverage());
assertEquals(2, platformMetrics.getMemory().getMinimum());
assertEquals(2, platformMetrics.getMemory().getMaximum());
assertEquals(2, platformMetrics.getMemory().getAverage());
assertNotNull(platformMetrics.getMemory().getUnit());

assertEquals(1, platformMetrics.getCost().getNumberOfDataPointsForAverage());
assertEquals(2, platformMetrics.getCost().getMinimum());
assertEquals(2, platformMetrics.getCost().getMaximum());
assertEquals(2, platformMetrics.getCost().getAverage());
assertNotNull(platformMetrics.getCost().getUnit());

assertEquals(1, platformMetrics.getExecutionTime().getNumberOfDataPointsForAverage());
assertEquals(300, platformMetrics.getExecutionTime().getMinimum());
assertEquals(300, platformMetrics.getExecutionTime().getMaximum());
assertEquals(300, platformMetrics.getExecutionTime().getAverage());
assertNotNull(platformMetrics.getExecutionTime().getUnit());

assertEquals(1, platformMetrics.getValidationStatus().getValidatorTools().size());
final String expectedValidatorTool = submittedValidationExecution.getValidatorTool().toString();
ValidatorInfo validationInfo = platformMetrics.getValidationStatus().getValidatorTools().get(expectedValidatorTool);
assertNotNull(validationInfo);
assertNotNull(validationInfo.getMostRecentVersionName());
ValidatorVersionInfo mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion1.equals(validationVersion.getName())).findFirst().get();
assertTrue(mostRecentValidationVersionInfo.isIsValid(), "miniwdl validation should be valid");
assertEquals(validatorToolVersion1, mostRecentValidationVersionInfo.getName());
assertEquals(100d, mostRecentValidationVersionInfo.getPassingRate());
assertEquals(1, mostRecentValidationVersionInfo.getNumberOfRuns());
assertEquals(100d, validationInfo.getPassingRate());
assertEquals(1, validationInfo.getNumberOfRuns());

Metrics platform2Metrics = version.getMetricsByPlatform().get(platform2);
assertNotNull(platform2Metrics);

// Verify that the aggregated metrics are the same as the single execution for platform2
assertEquals(1, platform2Metrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions());
assertEquals(0, platform2Metrics.getExecutionStatusCount().getNumberOfFailedExecutions());
assertEquals(1, platform2Metrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name()));
assertFalse(platform2Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name()));
assertFalse(platform2Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name()));

assertEquals(1, platform2Metrics.getCpu().getNumberOfDataPointsForAverage());
assertEquals(2, platform2Metrics.getCpu().getMinimum());
assertEquals(2, platform2Metrics.getCpu().getMaximum());
assertEquals(2, platform2Metrics.getCpu().getAverage());
assertNull(platform2Metrics.getCpu().getUnit());

assertEquals(1, platform2Metrics.getMemory().getNumberOfDataPointsForAverage());
assertEquals(2, platform2Metrics.getMemory().getMinimum());
assertEquals(2, platform2Metrics.getMemory().getMaximum());
assertEquals(2, platform2Metrics.getMemory().getAverage());
assertNotNull(platform2Metrics.getMemory().getUnit());

assertEquals(1, platform2Metrics.getExecutionTime().getNumberOfDataPointsForAverage());
assertEquals(300, platform2Metrics.getExecutionTime().getMinimum());
assertEquals(300, platform2Metrics.getExecutionTime().getMaximum());
assertEquals(300, platform2Metrics.getExecutionTime().getAverage());
assertNotNull(platform2Metrics.getExecutionTime().getUnit());

assertEquals(1, platform2Metrics.getValidationStatus().getValidatorTools().size());
validationInfo = platform2Metrics.getValidationStatus().getValidatorTools().get(WOMTOOL.toString());
assertNotNull(validationInfo);
assertNotNull(validationInfo.getMostRecentVersionName());
mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion2.equals(validationVersion.getName())).findFirst().get();
assertFalse(mostRecentValidationVersionInfo.isIsValid(), "womtool validation should be invalid");
assertEquals(validatorToolVersion2, mostRecentValidationVersionInfo.getName());
assertEquals(0d, mostRecentValidationVersionInfo.getPassingRate());
final String expectedMostRecentValidationVersionName = submittedValidationExecution.getValidatorToolVersion();
ValidatorVersionInfo mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> expectedMostRecentValidationVersionName.equals(validationVersion.getName())).findFirst().get();
assertEquals(submittedValidationExecution.isIsValid(), mostRecentValidationVersionInfo.isIsValid());
assertEquals(expectedMostRecentValidationVersionName, mostRecentValidationVersionInfo.getName());
assertEquals(1, mostRecentValidationVersionInfo.getNumberOfRuns());
assertEquals(0d, validationInfo.getPassingRate());
assertEquals(1, validationInfo.getNumberOfRuns());
if (submittedValidationExecution.isIsValid()) {
assertEquals(100d, mostRecentValidationVersionInfo.getPassingRate());
assertEquals(100d, validationInfo.getPassingRate());
} else {
assertEquals(0d, mostRecentValidationVersionInfo.getPassingRate());
assertEquals(0d, validationInfo.getPassingRate());
}
}

private static void testAggregatedMetrics(WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) {
private static void testOverallAggregatedMetrics(WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) {
ValidatorVersionInfo mostRecentValidationVersionInfo;
ValidatorInfo validationInfo;
// Verify that the metrics aggregated across ALL platforms are correct
Expand Down
6 changes: 4 additions & 2 deletions metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,14 @@ public final class TestUtilities {
private TestUtilities() {
}

public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB) {
public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB, Double costUSD, String region) {
return new RunExecution()
.executionStatus(executionStatus)
.executionTime(executionTime)
.cpuRequirements(cpuRequirements)
.memoryRequirementsGB(memoryRequirementsGB);
.memoryRequirementsGB(memoryRequirementsGB)
.costUSD(costUSD)
.region(region);
}

public static MetricsAggregatorConfig getMetricsConfig() {
Expand Down
Loading

0 comments on commit 379117d

Please sign in to comment.