From 379117dd792ed5e56448333328b411bdab1c7151 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 28 Jun 2023 09:55:46 -0400 Subject: [PATCH] Aggregate cost --- .../MetricsAggregatorS3Client.java | 2 + .../helper/AggregationHelper.java | 49 ++++++ .../MetricsAggregatorS3ClientIT.java | 2 +- .../client/cli/MetricsAggregatorClientIT.java | 139 ++++++++---------- .../common/TestUtilities.java | 6 +- .../helper/AggregationHelperTest.java | 37 +++++ pom.xml | 3 +- 7 files changed, 157 insertions(+), 81 deletions(-) diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java index 443222ae..09239967 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java @@ -73,6 +73,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) { return; } + System.out.println("Aggregating metrics..."); for (S3DirectoryInfo directoryInfo : metricsDirectories) { String toolId = directoryInfo.toolId(); String versionName = directoryInfo.versionId(); @@ -118,6 +119,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) { } } } + System.out.println("Completed aggregating metrics"); } /** diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java index a54d99a3..9e0931f0 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java @@ -5,6 +5,7 @@ import static java.util.stream.Collectors.groupingBy; import io.dockstore.metricsaggregator.Statistics; +import io.dockstore.openapi.client.model.CostMetric; import io.dockstore.openapi.client.model.CpuMetric; import io.dockstore.openapi.client.model.ExecutionStatusMetric; import io.dockstore.openapi.client.model.ExecutionTimeMetric; @@ -54,6 +55,7 @@ public static Optional getAggregatedMetrics(ExecutionsRequestBody allSu getAggregatedExecutionTime(allSubmissions).ifPresent(aggregatedMetrics::setExecutionTime); getAggregatedCpu(allSubmissions).ifPresent(aggregatedMetrics::setCpu); getAggregatedMemory(allSubmissions).ifPresent(aggregatedMetrics::setMemory); + getAggregatedCost(allSubmissions).ifPresent(aggregatedMetrics::setCost); } // Set validation metrics @@ -259,6 +261,53 @@ public static Optional getAggregatedMemoryFromExecutions(List getAggregatedCost(ExecutionsRequestBody allSubmissions) { + // Get aggregated Execution Time metrics that were submitted to Dockstore + List costMetrics = allSubmissions.getAggregatedExecutions().stream() + .map(Metrics::getCost) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + getAggregatedCostFromExecutions(allSubmissions.getRunExecutions()).ifPresent(costMetrics::add); + + if (!costMetrics.isEmpty()) { + List statistics = costMetrics.stream() + .map(metric -> new Statistics(metric.getMinimum(), metric.getMaximum(), metric.getAverage(), metric.getNumberOfDataPointsForAverage())).toList(); + Statistics newStatistic = Statistics.createFromStatistics(statistics); + return Optional.of(new CostMetric() + .minimum(newStatistic.min()) + .maximum(newStatistic.max()) + .average(newStatistic.average()) + .numberOfDataPointsForAverage(newStatistic.numberOfDataPoints())); + } + return Optional.empty(); + } + + /** + * Aggregate Cost metrics from the list of run executions by calculating the minimum, maximum, and average. + * @param executions + * @return + */ + public static Optional getAggregatedCostFromExecutions(List executions) { + List costs = executions.stream() + .map(RunExecution::getCostUSD) + .filter(Objects::nonNull) + .toList(); + if (!costs.isEmpty()) { + Statistics statistics = new Statistics(costs); + return Optional.of(new CostMetric() + .minimum(statistics.min()) + .maximum(statistics.max()) + .average(statistics.average()) + .numberOfDataPointsForAverage(statistics.numberOfDataPoints())); + } + return Optional.empty(); + } + /** * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of * each validator tool version. diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java index ac5d571b..8ffdef23 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java @@ -120,7 +120,7 @@ void testGetDirectories() { final String workflowVersionId = version.getName(); // A successful execution that ran for 5 minutes, requires 2 CPUs and 2 GBs of memory - RunExecution execution = createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0); + RunExecution execution = createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0, 2.00, "us-central1"); ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().runExecutions(List.of(execution)); extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform1, workflowId, workflowVersionId, ""); extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform2, workflowId, workflowVersionId, ""); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java index 9de43913..cceda10e 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java @@ -138,7 +138,7 @@ void testAggregateMetrics() { String versionId = version.getName(); // A successful run execution that ran for 5 minutes, requires 2 CPUs and 2 GBs of memory - List runExecutions = List.of(createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0)); + List runExecutions = List.of(createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0, 2.00, "us-central1")); // A successful miniwdl validation final String validatorToolVersion1 = "1.0"; ValidationExecution validationExecution1 = new ValidationExecution() @@ -164,15 +164,17 @@ void testAggregateMetrics() { version = workflow.getWorkflowVersions().stream().filter(v -> "master".equals(v.getName())).findFirst().orElse(null); assertNotNull(version); assertEquals(expectedNumberOfPlatforms, version.getMetricsByPlatform().size(), "There should be metrics for two platforms"); + assertAggregatedMetricsForPlatform(platform1, version, validationExecution1); + assertAggregatedMetricsForPlatform(platform2, version, validationExecution2); + Metrics platform1Metrics = version.getMetricsByPlatform().get(platform1); assertNotNull(platform1Metrics); - compareAggregateMetricsWithPlatforms(platform2, version, validatorToolVersion1, validatorToolVersion2, platform1Metrics); ValidatorVersionInfo mostRecentValidationVersionInfo; ValidatorInfo validationInfo; // A failed run execution that ran for 1 second, requires 2 CPUs and 4.5 GBs of memory - runExecutions = List.of(createRunExecution(FAILED_RUNTIME_INVALID, "PT1S", 4, 4.5)); + runExecutions = List.of(createRunExecution(FAILED_RUNTIME_INVALID, "PT1S", 4, 4.5, 2.00, "us-central1")); // A failed miniwdl validation for the same validator version List validationExecutions = List.of(new ValidationExecution().validatorTool(MINIWDL).validatorToolVersion("1.0").isValid(false).dateExecuted(Instant.now().toString())); ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().runExecutions(runExecutions).validationExecutions(validationExecutions); @@ -206,6 +208,12 @@ void testAggregateMetrics() { assertEquals(3.25, platform1Metrics.getMemory().getAverage()); assertNotNull(platform1Metrics.getMemory().getUnit()); + assertEquals(2, platform1Metrics.getCost().getNumberOfDataPointsForAverage()); + assertEquals(2, platform1Metrics.getCost().getMinimum()); + assertEquals(2, platform1Metrics.getCost().getMaximum()); + assertEquals(2, platform1Metrics.getCost().getAverage()); + assertNotNull(platform1Metrics.getCost().getUnit()); + assertEquals(2, platform1Metrics.getExecutionTime().getNumberOfDataPointsForAverage()); assertEquals(1, platform1Metrics.getExecutionTime().getMinimum()); assertEquals(300, platform1Metrics.getExecutionTime().getMaximum()); @@ -224,89 +232,66 @@ void testAggregateMetrics() { assertEquals(50d, validationInfo.getPassingRate()); assertEquals(2, validationInfo.getNumberOfRuns()); - testAggregatedMetrics(version, validatorToolVersion1, validatorToolVersion2, platform1Metrics); + testOverallAggregatedMetrics(version, validatorToolVersion1, validatorToolVersion2, platform1Metrics); } - private static void compareAggregateMetricsWithPlatforms(String platform2, WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) { - // Verify that the aggregated metrics are the same as the single execution for platform1 - assertEquals(1, platform1Metrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions()); - assertEquals(0, platform1Metrics.getExecutionStatusCount().getNumberOfFailedExecutions()); - assertEquals(1, platform1Metrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name())); - assertFalse(platform1Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name())); - assertFalse(platform1Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name())); - - assertEquals(1, platform1Metrics.getCpu().getNumberOfDataPointsForAverage()); - assertEquals(2, platform1Metrics.getCpu().getMinimum()); - assertEquals(2, platform1Metrics.getCpu().getMaximum()); - assertEquals(2, platform1Metrics.getCpu().getAverage()); - assertNull(platform1Metrics.getCpu().getUnit()); - - assertEquals(1, platform1Metrics.getMemory().getNumberOfDataPointsForAverage()); - assertEquals(2, platform1Metrics.getMemory().getMinimum()); - assertEquals(2, platform1Metrics.getMemory().getMaximum()); - assertEquals(2, platform1Metrics.getMemory().getAverage()); - assertNotNull(platform1Metrics.getMemory().getUnit()); - - assertEquals(1, platform1Metrics.getExecutionTime().getNumberOfDataPointsForAverage()); - assertEquals(300, platform1Metrics.getExecutionTime().getMinimum()); - assertEquals(300, platform1Metrics.getExecutionTime().getMaximum()); - assertEquals(300, platform1Metrics.getExecutionTime().getAverage()); - assertNotNull(platform1Metrics.getExecutionTime().getUnit()); - - assertEquals(1, platform1Metrics.getValidationStatus().getValidatorTools().size()); - ValidatorInfo validationInfo = platform1Metrics.getValidationStatus().getValidatorTools().get(MINIWDL.toString()); + private static void assertAggregatedMetricsForPlatform(String platform, WorkflowVersion version, ValidationExecution submittedValidationExecution) { + Metrics platformMetrics = version.getMetricsByPlatform().get(platform); + assertNotNull(platformMetrics); + + // Verify that the aggregated metrics are the same as the single execution for the platform + assertEquals(1, platformMetrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions()); + assertEquals(0, platformMetrics.getExecutionStatusCount().getNumberOfFailedExecutions()); + assertEquals(1, platformMetrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name())); + assertFalse(platformMetrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name())); + assertFalse(platformMetrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name())); + + assertEquals(1, platformMetrics.getCpu().getNumberOfDataPointsForAverage()); + assertEquals(2, platformMetrics.getCpu().getMinimum()); + assertEquals(2, platformMetrics.getCpu().getMaximum()); + assertEquals(2, platformMetrics.getCpu().getAverage()); + assertNull(platformMetrics.getCpu().getUnit()); + + assertEquals(1, platformMetrics.getMemory().getNumberOfDataPointsForAverage()); + assertEquals(2, platformMetrics.getMemory().getMinimum()); + assertEquals(2, platformMetrics.getMemory().getMaximum()); + assertEquals(2, platformMetrics.getMemory().getAverage()); + assertNotNull(platformMetrics.getMemory().getUnit()); + + assertEquals(1, platformMetrics.getCost().getNumberOfDataPointsForAverage()); + assertEquals(2, platformMetrics.getCost().getMinimum()); + assertEquals(2, platformMetrics.getCost().getMaximum()); + assertEquals(2, platformMetrics.getCost().getAverage()); + assertNotNull(platformMetrics.getCost().getUnit()); + + assertEquals(1, platformMetrics.getExecutionTime().getNumberOfDataPointsForAverage()); + assertEquals(300, platformMetrics.getExecutionTime().getMinimum()); + assertEquals(300, platformMetrics.getExecutionTime().getMaximum()); + assertEquals(300, platformMetrics.getExecutionTime().getAverage()); + assertNotNull(platformMetrics.getExecutionTime().getUnit()); + + assertEquals(1, platformMetrics.getValidationStatus().getValidatorTools().size()); + final String expectedValidatorTool = submittedValidationExecution.getValidatorTool().toString(); + ValidatorInfo validationInfo = platformMetrics.getValidationStatus().getValidatorTools().get(expectedValidatorTool); assertNotNull(validationInfo); assertNotNull(validationInfo.getMostRecentVersionName()); - ValidatorVersionInfo mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion1.equals(validationVersion.getName())).findFirst().get(); - assertTrue(mostRecentValidationVersionInfo.isIsValid(), "miniwdl validation should be valid"); - assertEquals(validatorToolVersion1, mostRecentValidationVersionInfo.getName()); - assertEquals(100d, mostRecentValidationVersionInfo.getPassingRate()); - assertEquals(1, mostRecentValidationVersionInfo.getNumberOfRuns()); - assertEquals(100d, validationInfo.getPassingRate()); - assertEquals(1, validationInfo.getNumberOfRuns()); - Metrics platform2Metrics = version.getMetricsByPlatform().get(platform2); - assertNotNull(platform2Metrics); - - // Verify that the aggregated metrics are the same as the single execution for platform2 - assertEquals(1, platform2Metrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions()); - assertEquals(0, platform2Metrics.getExecutionStatusCount().getNumberOfFailedExecutions()); - assertEquals(1, platform2Metrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name())); - assertFalse(platform2Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name())); - assertFalse(platform2Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name())); - - assertEquals(1, platform2Metrics.getCpu().getNumberOfDataPointsForAverage()); - assertEquals(2, platform2Metrics.getCpu().getMinimum()); - assertEquals(2, platform2Metrics.getCpu().getMaximum()); - assertEquals(2, platform2Metrics.getCpu().getAverage()); - assertNull(platform2Metrics.getCpu().getUnit()); - - assertEquals(1, platform2Metrics.getMemory().getNumberOfDataPointsForAverage()); - assertEquals(2, platform2Metrics.getMemory().getMinimum()); - assertEquals(2, platform2Metrics.getMemory().getMaximum()); - assertEquals(2, platform2Metrics.getMemory().getAverage()); - assertNotNull(platform2Metrics.getMemory().getUnit()); - - assertEquals(1, platform2Metrics.getExecutionTime().getNumberOfDataPointsForAverage()); - assertEquals(300, platform2Metrics.getExecutionTime().getMinimum()); - assertEquals(300, platform2Metrics.getExecutionTime().getMaximum()); - assertEquals(300, platform2Metrics.getExecutionTime().getAverage()); - assertNotNull(platform2Metrics.getExecutionTime().getUnit()); - - assertEquals(1, platform2Metrics.getValidationStatus().getValidatorTools().size()); - validationInfo = platform2Metrics.getValidationStatus().getValidatorTools().get(WOMTOOL.toString()); - assertNotNull(validationInfo); - assertNotNull(validationInfo.getMostRecentVersionName()); - mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion2.equals(validationVersion.getName())).findFirst().get(); - assertFalse(mostRecentValidationVersionInfo.isIsValid(), "womtool validation should be invalid"); - assertEquals(validatorToolVersion2, mostRecentValidationVersionInfo.getName()); - assertEquals(0d, mostRecentValidationVersionInfo.getPassingRate()); + final String expectedMostRecentValidationVersionName = submittedValidationExecution.getValidatorToolVersion(); + ValidatorVersionInfo mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> expectedMostRecentValidationVersionName.equals(validationVersion.getName())).findFirst().get(); + assertEquals(submittedValidationExecution.isIsValid(), mostRecentValidationVersionInfo.isIsValid()); + assertEquals(expectedMostRecentValidationVersionName, mostRecentValidationVersionInfo.getName()); assertEquals(1, mostRecentValidationVersionInfo.getNumberOfRuns()); - assertEquals(0d, validationInfo.getPassingRate()); assertEquals(1, validationInfo.getNumberOfRuns()); + if (submittedValidationExecution.isIsValid()) { + assertEquals(100d, mostRecentValidationVersionInfo.getPassingRate()); + assertEquals(100d, validationInfo.getPassingRate()); + } else { + assertEquals(0d, mostRecentValidationVersionInfo.getPassingRate()); + assertEquals(0d, validationInfo.getPassingRate()); + } } - private static void testAggregatedMetrics(WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) { + private static void testOverallAggregatedMetrics(WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) { ValidatorVersionInfo mostRecentValidationVersionInfo; ValidatorInfo validationInfo; // Verify that the metrics aggregated across ALL platforms are correct diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java index 73813452..fc02daa2 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java @@ -34,12 +34,14 @@ public final class TestUtilities { private TestUtilities() { } - public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB) { + public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB, Double costUSD, String region) { return new RunExecution() .executionStatus(executionStatus) .executionTime(executionTime) .cpuRequirements(cpuRequirements) - .memoryRequirementsGB(memoryRequirementsGB); + .memoryRequirementsGB(memoryRequirementsGB) + .costUSD(costUSD) + .region(region); } public static MetricsAggregatorConfig getMetricsConfig() { diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java index 624c2646..7f7f60e4 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.dockstore.openapi.client.model.CostMetric; import io.dockstore.openapi.client.model.CpuMetric; import io.dockstore.openapi.client.model.ExecutionStatusMetric; import io.dockstore.openapi.client.model.ExecutionTimeMetric; @@ -166,6 +167,42 @@ void testGetAggregatedMemory() { assertEquals(3, memoryMetric.get().getNumberOfDataPointsForAverage()); } + @Test + void testGetAggregatedCost() { + List executions = new ArrayList<>(); + Optional costMetric = AggregationHelper.getAggregatedCost(new ExecutionsRequestBody().runExecutions(executions)); + assertTrue(costMetric.isEmpty()); + + // Add an execution that doesn't have cost data + executions.add(new RunExecution().executionStatus(SUCCESSFUL)); + costMetric = AggregationHelper.getAggregatedCost(new ExecutionsRequestBody().runExecutions(executions)); + assertTrue(costMetric.isEmpty()); + + // Add an execution with cost data + Double costInUSD = 2.00; + executions.add(new RunExecution().executionStatus(SUCCESSFUL).costUSD(costInUSD)); + costMetric = AggregationHelper.getAggregatedCost(new ExecutionsRequestBody().runExecutions(executions)); + assertTrue(costMetric.isPresent()); + assertEquals(costInUSD, costMetric.get().getMinimum()); + assertEquals(costInUSD, costMetric.get().getMaximum()); + assertEquals(costInUSD, costMetric.get().getAverage()); + assertEquals(1, costMetric.get().getNumberOfDataPointsForAverage()); + + // Aggregate submissions containing run executions and aggregated metrics + Metrics submittedAggregatedMetrics = new Metrics() + .cost(new CostMetric() + .minimum(2.00) + .maximum(6.00) + .average(4.00) + .numberOfDataPointsForAverage(2)); + costMetric = AggregationHelper.getAggregatedCost(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); + assertTrue(costMetric.isPresent()); + assertEquals(2.0, costMetric.get().getMinimum()); + assertEquals(6.0, costMetric.get().getMaximum()); + assertEquals(3.333333333333333, costMetric.get().getAverage()); + assertEquals(3, costMetric.get().getNumberOfDataPointsForAverage()); + } + @Test void testGetAggregatedValidationStatus() { List executions = new ArrayList<>(); diff --git a/pom.xml b/pom.xml index 04b33f9e..14025c29 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,8 @@ scm:git:git@github.com:dockstore/dockstore-support.git UTF-8 - 1.15.0-alpha.4 + + 1.15.0-SNAPSHOT 3.0.0-M5 2.22.2 false