diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index f56d7df6..34edfae9 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -1,5 +1,5 @@ -Lists of 399 third-party dependencies. +Lists of 402 third-party dependencies. (Apache License, Version 2.0) akka-actor (com.typesafe.akka:akka-actor_2.13:2.5.32 - https://akka.io/) (Apache License, Version 2.0) akka-protobuf (com.typesafe.akka:akka-protobuf_2.13:2.5.32 - https://akka.io/) (Apache License, Version 2.0) akka-slf4j (com.typesafe.akka:akka-slf4j_2.13:2.5.32 - https://akka.io/) @@ -124,10 +124,10 @@ Lists of 399 third-party dependencies. (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.0 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.0 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport-httpclient5 (com.github.docker-java:docker-java-transport-httpclient5:3.3.0 - https://github.com/docker-java/docker-java) - (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-alpha.4 - no url defined) - (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-alpha.4 - no url defined) - (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-alpha.4 - no url defined) - (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-alpha.4 - no url defined) + (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-alpha.5 - no url defined) + (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-alpha.5 - no url defined) + (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-alpha.5 - no url defined) + (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-alpha.5 - no url defined) (Apache License 2.0) Dropwizard (io.dropwizard:dropwizard-core:4.0.0 - http://www.dropwizard.io/4.0.0/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-core) (Apache License 2.0) Dropwizard Asset Bundle (io.dropwizard:dropwizard-assets:4.0.0 - http://www.dropwizard.io/4.0.0/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-assets) (Apache License 2.0) Dropwizard Authentication (io.dropwizard:dropwizard-auth:4.0.0 - http://www.dropwizard.io/4.0.0/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-auth) @@ -228,6 +228,7 @@ Lists of 399 third-party dependencies. (The Apache Software License, Version 2.0) java-diff-utils (io.github.java-diff-utils:java-diff-utils:4.12 - https://github.com/java-diff-utils/java-diff-utils/java-diff-utils) (CDDL/GPLv2+CE) JavaBeans Activation Framework (com.sun.activation:javax.activation:1.2.0 - http://java.net/all/javax.activation/) (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.29.2-GA - http://www.javassist.org/) + (CDDL + GPLv2 with classpath exception) javax.annotation API (javax.annotation:javax.annotation-api:1.3.2 - http://jcp.org/en/jsr/detail?id=250) (Eclipse Distribution License - v 1.0) JAXB Core (org.glassfish.jaxb:jaxb-core:3.0.2 - https://eclipse-ee4j.github.io/jaxb-ri/) (Eclipse Distribution License - v 1.0) JAXB Runtime (org.glassfish.jaxb:jaxb-runtime:3.0.2 - https://eclipse-ee4j.github.io/jaxb-ri/) (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.3.Final - http://www.jboss.org) @@ -278,6 +279,7 @@ Lists of 399 third-party dependencies. (Public Domain) JSON in Java (org.json:json:20230227 - https://github.com/douglascrockford/JSON-java) (Revised BSD License) JSONLD Java :: Core (com.github.jsonld-java:jsonld-java:0.8.3 - http://github.com/jsonld-java/jsonld-java/jsonld-java/) (The MIT License) jsoup Java HTML Parser (org.jsoup:jsoup:1.10.2 - https://jsoup.org/) + (Apache License, Version 2.0) JSR 354 (Money and Currency API) (javax.money:money-api:1.1 - https://javamoney.github.io/) (MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:2.0.7 - http://www.slf4j.org) (Eclipse Public License 1.0) JUnit (junit:junit:4.13.2 - http://junit.org) (Eclipse Public License v2.0) JUnit Jupiter (Aggregator) (org.junit.jupiter:junit-jupiter:5.9.2 - https://junit.org/junit5/) @@ -318,6 +320,7 @@ Lists of 399 third-party dependencies. (Eclipse Distribution License - v 1.0) MIME streaming extension (org.jvnet.mimepull:mimepull:1.9.15 - https://github.com/eclipse-ee4j/metro-mimepull) (The MIT License) mockito-core (org.mockito:mockito-core:3.12.4 - https://github.com/mockito/mockito) (The MIT License) mockito-inline (org.mockito:mockito-inline:3.12.4 - https://github.com/mockito/mockito) + (Apache 2 License) Moneta Core (org.javamoney.moneta:moneta-core:1.4.2 - http://javamoney.org) (MIT license) mouse (org.typelevel:mouse_2.13:1.0.10 - https://typelevel.org/mouse) (Apache License, Version 2.0) Netty Reactive Streams HTTP support (com.typesafe.netty:netty-reactive-streams-http:2.0.5 - https://github.com/playframework/netty-reactive-streams/netty-reactive-streams-http) (Apache License, Version 2.0) Netty Reactive Streams Implementation (com.typesafe.netty:netty-reactive-streams:2.0.5 - https://github.com/playframework/netty-reactive-streams/netty-reactive-streams) @@ -340,7 +343,7 @@ Lists of 399 third-party dependencies. (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis) (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/) (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/) - (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-alpha.4 - no url defined) + (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-alpha.5 - no url defined) (The Apache License, Version 2.0) OpenCensus (io.opencensus:opencensus-api:0.31.0 - https://github.com/census-instrumentation/opencensus-java) (Apache 2) opencsv (com.opencsv:opencsv:5.7.1 - http://opencsv.sf.net) (MIT License) ORCID - Model (org.orcid:orcid-model-jakarta:3.3.0 - https://github.com/ORCID/orcid-model) @@ -378,7 +381,7 @@ Lists of 399 third-party dependencies. (Apache License 2.0) swagger-core-jakarta (io.swagger.core.v3:swagger-core-jakarta:2.2.9 - https://github.com/swagger-api/swagger-core/modules/swagger-core-jakarta) (Apache License 2.0) swagger-integration-jakarta (io.swagger.core.v3:swagger-integration-jakarta:2.2.9 - https://github.com/swagger-api/swagger-core/modules/swagger-integration-jakarta) (Apache Software License, Version 2.0) swagger-java-bitbucket-client (io.dockstore:swagger-java-bitbucket-client:2.0.3 - no url defined) - (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-alpha.4 - no url defined) + (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-alpha.5 - no url defined) (Apache Software License, Version 2.0) swagger-java-discourse-client (io.dockstore:swagger-java-discourse-client:2.0.1 - no url defined) (Apache Software License, Version 2.0) swagger-java-quay-client (io.dockstore:swagger-java-quay-client:2.0.2 - no url defined) (Apache Software License, Version 2.0) swagger-java-sam-client (io.dockstore:swagger-java-sam-client:2.0.2 - no url defined) diff --git a/metricsaggregator/pom.xml b/metricsaggregator/pom.xml index 0d16375c..3d4ee2f0 100644 --- a/metricsaggregator/pom.xml +++ b/metricsaggregator/pom.xml @@ -131,6 +131,14 @@ software.amazon.awssdk s3 + + org.javamoney.moneta + moneta-core + + + javax.money + money-api + org.junit.jupiter junit-jupiter-api @@ -263,6 +271,8 @@ org.slf4j:slf4j-api software.amazon.awssdk:s3 org.glassfish.jersey.inject:jersey-hk2 + javax.money:money-api + org.javamoney.moneta:moneta-core diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/DoubleStatistics.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/DoubleStatistics.java new file mode 100644 index 00000000..a5604573 --- /dev/null +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/DoubleStatistics.java @@ -0,0 +1,114 @@ +/* + * Copyright 2023 OICR and UCSC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.dockstore.metricsaggregator; + +import java.util.List; + +/** + * Record that contains statistical information obtained from a list of Doubles. + */ +public class DoubleStatistics extends Statistics { + public DoubleStatistics() { + super(); + } + + /** + * Constructor that calculates statistical information from the provided list of data points. + * @param dataPoints List of Doubles + */ + public DoubleStatistics(List dataPoints) { + super(dataPoints); + } + + public DoubleStatistics(Double minimum, Double maximum, Double average, int numberOfDataPoints) { + super(minimum, maximum, average, numberOfDataPoints); + } + + /** + * Constructor used to create a Statistics object that can be used to calculate weighted averages for non-Statistics objects. + * A placeholder value is set for the min and maximum fields + * @param average + * @param numberOfDataPoints + */ + public DoubleStatistics(double average, int numberOfDataPoints) { + super(0d, 0d, average, numberOfDataPoints); + } + + /** + * Create a new Statistics object from a list of statistics by aggregating the list of statistics + * @param statistics + * @return + */ + public static DoubleStatistics createFromStatistics(List statistics) { + if (statistics.size() == 1) { + return statistics.get(0); + } + + DoubleStatistics newStatistics = new DoubleStatistics(); + newStatistics.setAverage(statistics); + newStatistics.setMinimum(statistics); + newStatistics.setMaximum(statistics); + newStatistics.setNumberOfDataPoints(statistics); + return newStatistics; + } + + /** + * Get the lowest value from the list of data points. + * @param dataPoints + * @return + */ + @Override + public Double calculateMinimum(List dataPoints) { + return dataPoints.stream().mapToDouble(d -> d).min().orElse(0); + } + + /** + * Get the highest value from the list of data points. + * @param dataPoints + * @return + */ + @Override + public Double calculateMaximum(List dataPoints) { + return dataPoints.stream().mapToDouble(d -> d).max().orElse(0); + } + + /** + * Calculate the average from the list of data points. + * @param dataPoints + * @return + */ + @Override + public Double calculateAverage(List dataPoints) { + return dataPoints.stream().mapToDouble(d -> d).average().orElse(0); + } + + /** + * Calculate a weighted average + */ + @Override + public Double calculateWeightedAverage(List> statistics) { + int totalNumberOfDataPoints = getTotalNumberOfDataPoints(statistics); + return statistics.stream() + .map(stat -> { + double weight = (double)stat.getNumberOfDataPoints() / (double)totalNumberOfDataPoints; + return stat.getAverage() * weight; + }) + .mapToDouble(Double::doubleValue) + .sum(); + } +} diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java index 443222ae..09239967 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java @@ -73,6 +73,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) { return; } + System.out.println("Aggregating metrics..."); for (S3DirectoryInfo directoryInfo : metricsDirectories) { String toolId = directoryInfo.toolId(); String versionName = directoryInfo.versionId(); @@ -118,6 +119,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) { } } } + System.out.println("Completed aggregating metrics"); } /** diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MoneyStatistics.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MoneyStatistics.java new file mode 100644 index 00000000..87102583 --- /dev/null +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MoneyStatistics.java @@ -0,0 +1,90 @@ +package io.dockstore.metricsaggregator; + +import java.util.List; +import org.javamoney.moneta.Money; + +/** + * Calculates money statistics in USD using the Java Money library to preserve accuracy. + */ +public class MoneyStatistics extends Statistics { + private static String currency = "USD"; + + private MoneyStatistics() { + super(); + } + + public MoneyStatistics(List dataPoints) { + super(dataPoints); + } + + public MoneyStatistics(Money minimum, Money maximum, Money average, int numberOfDataPoints) { + super(minimum, maximum, average, numberOfDataPoints); + } + + /** + * Create a new Statistics object from a list of statistics by aggregating the list of statistics + * @param statistics + * @return + */ + public static MoneyStatistics createFromStatistics(List statistics) { + if (statistics.size() == 1) { + return statistics.get(0); + } + + MoneyStatistics newStatistics = new MoneyStatistics(); + newStatistics.setAverage(statistics); + newStatistics.setMinimum(statistics); + newStatistics.setMaximum(statistics); + newStatistics.setNumberOfDataPoints(statistics); + return newStatistics; + } + + /** + * Get the lowest value from the list of data points. + * @param dataPoints + * @return + */ + @Override + public Money calculateMinimum(List dataPoints) { + return dataPoints.stream() + .min(Money::compareTo) + .orElse(Money.of(0, currency)); + } + + /** + * Get the highest value from the list of data points. + * @param dataPoints + * @return + */ + @Override + public Money calculateMaximum(List dataPoints) { + return dataPoints.stream() + .max(Money::compareTo) + .orElse(Money.of(0, currency)); + } + + /** + * Calculate the average from the list of data points. + * @param dataPoints + * @return + */ + @Override + public Money calculateAverage(List dataPoints) { + Money sum = dataPoints.stream().reduce(Money.of(0, currency), Money::add); + return sum.divide(dataPoints.size()); + } + + /** + * Calculate a weighted average + */ + @Override + public Money calculateWeightedAverage(List> statistics) { + int totalNumberOfDataPoints = getTotalNumberOfDataPoints(statistics); + return statistics.stream() + .map(stat -> { + double weight = (double)stat.getNumberOfDataPoints() / (double)totalNumberOfDataPoints; + return stat.getAverage().multiply(weight); + }) + .reduce(Money.of(0, currency), Money::add); + } +} diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/Statistics.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/Statistics.java index 4f28fa22..6e7e5fe2 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/Statistics.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/Statistics.java @@ -1,108 +1,104 @@ -/* - * Copyright 2023 OICR and UCSC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - package io.dockstore.metricsaggregator; import java.util.List; -import java.util.stream.Stream; /** - * Record that contains statistical information obtained from a list of Doubles. - * @param min - * @param max - * @param average - * @param numberOfDataPoints + * A class that contains statistical information for a data type. + * @param */ -public record Statistics(double min, double max, double average, int numberOfDataPoints) { - /** - * Constructor that calculates statistical information from the provided list of data points. - * @param dataPoints List of Doubles - */ - public Statistics(List dataPoints) { - this(getMinimum(dataPoints), getMaximum(dataPoints), getAverage(dataPoints), dataPoints.size()); +public abstract class Statistics { + private T minimum; + private T maximum; + private T average; + private int numberOfDataPoints; + + protected Statistics() { } - /** - * Constructor used to create a Statistics object that can be used to calculate weighted averages for non-Statistics objects. - * A placeholder value is set for the min and maximum fields - * @param average - * @param numberOfDataPoints - */ - public Statistics(double average, int numberOfDataPoints) { - this(0, 0, average, numberOfDataPoints); + protected Statistics(T minimum, T maximum, T average, int numberOfDataPoints) { + this.minimum = minimum; + this.maximum = maximum; + this.average = average; + this.numberOfDataPoints = numberOfDataPoints; } - /** - * Create a new Statistics object from a list of statistics by aggregating the list of statistics - * @param statistics - * @return - */ - public static Statistics createFromStatistics(List statistics) { - if (statistics.size() == 1) { - return statistics.get(0); - } - List dataPoints = statistics.stream() - .flatMap(stat -> Stream.of(stat.min(), stat.max())) + protected Statistics(List dataPoints) { + this.minimum = calculateMinimum(dataPoints); + this.maximum = calculateMaximum(dataPoints); + this.average = calculateAverage(dataPoints); + this.numberOfDataPoints = dataPoints.size(); + } + + public abstract T calculateMinimum(List dataPoints); + public abstract T calculateMaximum(List dataPoints); + public abstract T calculateAverage(List dataPoints); + public abstract T calculateWeightedAverage(List> statistics); + + public T getMinimum() { + return minimum; + } + + public void setMinimum(T min) { + this.minimum = min; + } + + public void setMinimum(List> statistics) { + List dataPoints = statistics.stream() + .map(Statistics::getMinimum) .toList(); - double min = getMinimum(dataPoints); - double max = getMaximum(dataPoints); - double average = getWeightedAverage(statistics); - int numberOfDataPoints = statistics.stream().map(Statistics::numberOfDataPoints).mapToInt(Integer::intValue).sum(); - return new Statistics(min, max, average, numberOfDataPoints); + this.minimum = calculateMinimum(dataPoints); } - /** - * Get the lowest value from the list of data points. - * @param dataPoints - * @return - */ - public static double getMinimum(List dataPoints) { - return dataPoints.stream().mapToDouble(d -> d).min().getAsDouble(); + public T getMaximum() { + return maximum; } - /** - * Get the highest value from the list of data points. - * @param dataPoints - * @return - */ - public static double getMaximum(List dataPoints) { - return dataPoints.stream().mapToDouble(d -> d).max().getAsDouble(); + public void setMaximum(T max) { + this.maximum = max; } - /** - * Calculate the average from the list of data points. - * @param dataPoints - * @return - */ - public static double getAverage(List dataPoints) { - return dataPoints.stream().mapToDouble(d -> d).average().getAsDouble(); + public void setMaximum(List> statistics) { + List dataPoints = statistics.stream() + .map(Statistics::getMaximum) + .toList(); + this.maximum = calculateMaximum(dataPoints); + } + + public T getAverage() { + return average; + } + + public void setAverage(T average) { + this.average = average; } /** - * Calculate a weighted average + * Sets the average by calculating the weighted average from a list of statistics + * @param statistics */ - public static double getWeightedAverage(List statistics) { - int totalNumberOfDataPoints = statistics.stream().map(Statistics::numberOfDataPoints).mapToInt(Integer::intValue).sum(); + public void setAverage(List> statistics) { + this.average = calculateWeightedAverage(statistics); + } + + public int getNumberOfDataPoints() { + return numberOfDataPoints; + } + + public void setNumberOfDataPoints(int numberOfDataPoints) { + this.numberOfDataPoints = numberOfDataPoints; + } + + public void setNumberOfDataPoints(List> statistics) { + this.numberOfDataPoints = statistics.stream() + .map(Statistics::getNumberOfDataPoints) + .mapToInt(Integer::intValue) + .sum(); + } + + public int getTotalNumberOfDataPoints(List> statistics) { return statistics.stream() - .map(stat -> { - double weight = (double)stat.numberOfDataPoints() / (double)totalNumberOfDataPoints; - return stat.average() * weight; - }) - .mapToDouble(Double::doubleValue) + .map(Statistics::getNumberOfDataPoints) + .mapToInt(Integer::intValue) .sum(); } } diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java index 05c06e9e..b03cc7ca 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java @@ -196,8 +196,8 @@ private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolE ValidationExecution validationExecution = new ValidationExecution() .validatorTool(validator) .validatorToolVersion(validatorVersion) - .isValid(isValid) - .dateExecuted(dateExecuted); + .isValid(isValid); + validationExecution.setDateExecuted(dateExecuted); ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().validationExecutions(List.of(validationExecution)); try { diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java index a54d99a3..8041f441 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java @@ -2,9 +2,13 @@ import static io.dockstore.common.metrics.FormatCheckHelper.checkExecutionDateISO8601Format; import static io.dockstore.common.metrics.FormatCheckHelper.checkExecutionTimeISO8601Format; +import static io.dockstore.common.metrics.FormatCheckHelper.isValidCurrencyCode; import static java.util.stream.Collectors.groupingBy; -import io.dockstore.metricsaggregator.Statistics; +import io.dockstore.metricsaggregator.DoubleStatistics; +import io.dockstore.metricsaggregator.MoneyStatistics; +import io.dockstore.openapi.client.model.Cost; +import io.dockstore.openapi.client.model.CostMetric; import io.dockstore.openapi.client.model.CpuMetric; import io.dockstore.openapi.client.model.ExecutionStatusMetric; import io.dockstore.openapi.client.model.ExecutionTimeMetric; @@ -17,6 +21,7 @@ import io.dockstore.openapi.client.model.ValidatorInfo; import io.dockstore.openapi.client.model.ValidatorVersionInfo; import java.time.Duration; +import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.Date; @@ -29,6 +34,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.javamoney.moneta.Money; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +60,7 @@ public static Optional getAggregatedMetrics(ExecutionsRequestBody allSu getAggregatedExecutionTime(allSubmissions).ifPresent(aggregatedMetrics::setExecutionTime); getAggregatedCpu(allSubmissions).ifPresent(aggregatedMetrics::setCpu); getAggregatedMemory(allSubmissions).ifPresent(aggregatedMetrics::setMemory); + getAggregatedCost(allSubmissions).ifPresent(aggregatedMetrics::setCost); } // Set validation metrics @@ -111,18 +118,20 @@ public static Optional getAggregatedExecutionTime(Execution List executionTimeMetrics = allSubmissions.getAggregatedExecutions().stream() .map(Metrics::getExecutionTime) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .collect(Collectors.toCollection(ArrayList::new)); getAggregatedExecutionTimeFromExecutions(allSubmissions.getRunExecutions()).ifPresent(executionTimeMetrics::add); if (!executionTimeMetrics.isEmpty()) { - List statistics = executionTimeMetrics.stream() - .map(metric -> new Statistics(metric.getMinimum(), metric.getMaximum(), metric.getAverage(), metric.getNumberOfDataPointsForAverage())).toList(); - Statistics newStatistic = Statistics.createFromStatistics(statistics); + List statistics = executionTimeMetrics.stream() + .map(metric -> new DoubleStatistics(metric.getMinimum(), metric.getMaximum(), metric.getAverage(), metric.getNumberOfDataPointsForAverage())) + .toList(); + + DoubleStatistics newStatistic = DoubleStatistics.createFromStatistics(statistics); return Optional.of(new ExecutionTimeMetric() - .minimum(newStatistic.min()) - .maximum(newStatistic.max()) - .average(newStatistic.average()) - .numberOfDataPointsForAverage(newStatistic.numberOfDataPoints())); + .minimum(newStatistic.getMinimum()) + .maximum(newStatistic.getMaximum()) + .average(newStatistic.getAverage()) + .numberOfDataPointsForAverage(newStatistic.getNumberOfDataPoints())); } return Optional.empty(); @@ -154,12 +163,12 @@ public static Optional getAggregatedExecutionTimeFromExecut .toList(); if (!executionTimesInSeconds.isEmpty()) { - Statistics statistics = new Statistics(executionTimesInSeconds); + DoubleStatistics statistics = new DoubleStatistics(executionTimesInSeconds); return Optional.of(new ExecutionTimeMetric() - .minimum(statistics.min()) - .maximum(statistics.max()) - .average(statistics.average()) - .numberOfDataPointsForAverage(statistics.numberOfDataPoints())); + .minimum(statistics.getMinimum()) + .maximum(statistics.getMaximum()) + .average(statistics.getAverage()) + .numberOfDataPointsForAverage(statistics.getNumberOfDataPoints())); } return Optional.empty(); } @@ -174,18 +183,18 @@ public static Optional getAggregatedCpu(ExecutionsRequestBody allSubm List cpuMetrics = allSubmissions.getAggregatedExecutions().stream() .map(Metrics::getCpu) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .collect(Collectors.toCollection(ArrayList::new)); getAggregatedCpuFromExecutions(allSubmissions.getRunExecutions()).ifPresent(cpuMetrics::add); if (!cpuMetrics.isEmpty()) { - List statistics = cpuMetrics.stream() - .map(metric -> new Statistics(metric.getMinimum(), metric.getMaximum(), metric.getAverage(), metric.getNumberOfDataPointsForAverage())).toList(); - Statistics newStatistic = Statistics.createFromStatistics(statistics); + List statistics = cpuMetrics.stream() + .map(metric -> new DoubleStatistics(metric.getMinimum(), metric.getMaximum(), metric.getAverage(), metric.getNumberOfDataPointsForAverage())).toList(); + DoubleStatistics newStatistic = DoubleStatistics.createFromStatistics(statistics); return Optional.of(new CpuMetric() - .minimum(newStatistic.min()) - .maximum(newStatistic.max()) - .average(newStatistic.average()) - .numberOfDataPointsForAverage(newStatistic.numberOfDataPoints())); + .minimum(newStatistic.getMinimum()) + .maximum(newStatistic.getMaximum()) + .average(newStatistic.getAverage()) + .numberOfDataPointsForAverage(newStatistic.getNumberOfDataPoints())); } return Optional.empty(); } @@ -202,12 +211,12 @@ public static Optional getAggregatedCpuFromExecutions(List getAggregatedMemory(ExecutionsRequestBody a List memoryMetrics = allSubmissions.getAggregatedExecutions().stream() .map(Metrics::getMemory) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .collect(Collectors.toCollection(ArrayList::new)); getAggregatedMemoryFromExecutions(allSubmissions.getRunExecutions()).ifPresent(memoryMetrics::add); if (!memoryMetrics.isEmpty()) { - List statistics = memoryMetrics.stream() - .map(metric -> new Statistics(metric.getMinimum(), metric.getMaximum(), metric.getAverage(), metric.getNumberOfDataPointsForAverage())).toList(); - Statistics newStatistic = Statistics.createFromStatistics(statistics); + List statistics = memoryMetrics.stream() + .map(metric -> new DoubleStatistics(metric.getMinimum(), metric.getMaximum(), metric.getAverage(), metric.getNumberOfDataPointsForAverage())).toList(); + DoubleStatistics newStatistic = DoubleStatistics.createFromStatistics(statistics); return Optional.of(new MemoryMetric() - .minimum(newStatistic.min()) - .maximum(newStatistic.max()) - .average(newStatistic.average()) - .numberOfDataPointsForAverage(newStatistic.numberOfDataPoints())); + .minimum(newStatistic.getMinimum()) + .maximum(newStatistic.getMaximum()) + .average(newStatistic.getAverage()) + .numberOfDataPointsForAverage(newStatistic.getNumberOfDataPoints())); } return Optional.empty(); } @@ -249,12 +258,71 @@ public static Optional getAggregatedMemoryFromExecutions(List getAggregatedCost(ExecutionsRequestBody allSubmissions) { + // Get aggregated cost metrics that were submitted to Dockstore + List costMetrics = allSubmissions.getAggregatedExecutions().stream() + .map(Metrics::getCost) + .filter(Objects::nonNull) + .collect(Collectors.toCollection(ArrayList::new)); + getAggregatedCostFromExecutions(allSubmissions.getRunExecutions()).ifPresent(costMetrics::add); + + if (!costMetrics.isEmpty()) { + List statistics = costMetrics.stream() + .map(metric -> new MoneyStatistics(Money.of(metric.getMinimum(), metric.getUnit()), Money.of(metric.getMaximum(), metric.getUnit()), Money.of(metric.getAverage(), + metric.getUnit()), metric.getNumberOfDataPointsForAverage())) + .toList(); + MoneyStatistics moneyStatistics = MoneyStatistics.createFromStatistics(statistics); + return Optional.of(new CostMetric() + .minimum(moneyStatistics.getMinimum().getNumber().doubleValue()) + .maximum(moneyStatistics.getMaximum().getNumber().doubleValue()) + .average(moneyStatistics.getAverage().getNumber().doubleValue()) + .numberOfDataPointsForAverage(moneyStatistics.getNumberOfDataPoints())); + } + return Optional.empty(); + } + + /** + * Aggregate Cost metrics from the list of run executions by calculating the minimum, maximum, and average. + * @param executions + * @return + */ + public static Optional getAggregatedCostFromExecutions(List executions) { + List submittedCosts = executions.stream() + .map(RunExecution::getCost) + .filter(Objects::nonNull) + .toList(); + + boolean containsMalformedCurrencies = submittedCosts.stream().anyMatch(cost -> !isValidCurrencyCode(cost.getCurrency())); + // This shouldn't happen until we allow users to submit any currency they want + if (containsMalformedCurrencies) { + return Optional.empty(); // Don't aggregate if there's malformed data + } + + if (!submittedCosts.isEmpty()) { + List costs = submittedCosts.stream() + .map(cost -> Money.of(cost.getValue(), cost.getCurrency())) + .toList(); + MoneyStatistics statistics = new MoneyStatistics(costs); + return Optional.of(new CostMetric() + .minimum(statistics.getMinimum().getNumber().doubleValue()) + .maximum(statistics.getMaximum().getNumber().doubleValue()) + .average(statistics.getAverage().getNumber().doubleValue()) + .numberOfDataPointsForAverage(statistics.getNumberOfDataPoints())); } return Optional.empty(); } @@ -270,7 +338,7 @@ public static Optional getAggregatedValidationStatus(Exe List validationStatusMetrics = allSubmissions.getAggregatedExecutions().stream() .map(Metrics::getValidationStatus) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .collect(Collectors.toCollection(ArrayList::new)); getAggregatedValidationStatusFromExecutions(allSubmissions.getValidationExecutions()).ifPresent(validationStatusMetrics::add); Map newValidatorToolToValidatorInfo = new HashMap<>(); @@ -290,9 +358,11 @@ public static Optional getAggregatedValidationStatus(Exe final int numberOfRuns = validationVersionInfosByValidatorTool.stream().map(ValidatorVersionInfo::getNumberOfRuns) .mapToInt(Integer::intValue) .sum(); - final double passingRate = Statistics.getWeightedAverage(validationVersionInfosByValidatorTool.stream() - .map(validatorVersionInfo -> new Statistics(validatorVersionInfo.getPassingRate(), validatorVersionInfo.getNumberOfRuns())) - .toList()); + final List validationRunsStatistics = validationVersionInfosByValidatorTool.stream() + .map(validatorVersionInfo -> new DoubleStatistics(validatorVersionInfo.getPassingRate(), validatorVersionInfo.getNumberOfRuns())) + .toList(); + + final double passingRate = DoubleStatistics.createFromStatistics(validationRunsStatistics).getAverage(); final Optional mostRecentValidationVersion = getLatestValidationVersionInfo(validationVersionInfosByValidatorTool); if (mostRecentValidationVersion.isPresent()) { diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/StatisticsTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/DoubleStatisticsTest.java similarity index 73% rename from metricsaggregator/src/test/java/io/dockstore/metricsaggregator/StatisticsTest.java rename to metricsaggregator/src/test/java/io/dockstore/metricsaggregator/DoubleStatisticsTest.java index 696d28ad..f3c6c8d3 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/StatisticsTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/DoubleStatisticsTest.java @@ -22,15 +22,15 @@ import java.util.List; import org.junit.jupiter.api.Test; -class StatisticsTest { +class DoubleStatisticsTest { @Test void testStatistic() { List fibonacci = List.of(0.0, 1.0, 1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 21.0, 34.0, 55.0, 89.0, 144.0); - Statistics statistics = new Statistics(fibonacci); - assertEquals(0, statistics.min()); - assertEquals(144, statistics.max()); - assertEquals(29, Math.round(statistics.average())); - assertEquals(13, statistics.numberOfDataPoints()); + DoubleStatistics statistics = new DoubleStatistics(fibonacci); + assertEquals(0, statistics.getMinimum()); + assertEquals(144, statistics.getMaximum()); + assertEquals(29, Math.round(statistics.getAverage())); + assertEquals(13, statistics.getNumberOfDataPoints()); } } diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java index ac5d571b..bb0ebb28 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MetricsAggregatorS3ClientIT.java @@ -40,6 +40,7 @@ import io.dockstore.openapi.client.api.ContainertagsApi; import io.dockstore.openapi.client.api.ExtendedGa4GhApi; import io.dockstore.openapi.client.api.WorkflowsApi; +import io.dockstore.openapi.client.model.Cost; import io.dockstore.openapi.client.model.DockstoreTool; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.RunExecution; @@ -120,7 +121,7 @@ void testGetDirectories() { final String workflowVersionId = version.getName(); // A successful execution that ran for 5 minutes, requires 2 CPUs and 2 GBs of memory - RunExecution execution = createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0); + RunExecution execution = createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0, new Cost().value(2.00), "us-central1"); ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().runExecutions(List.of(execution)); extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform1, workflowId, workflowVersionId, ""); extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform2, workflowId, workflowVersionId, ""); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MoneyStatisticsTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MoneyStatisticsTest.java new file mode 100644 index 00000000..1e3f7138 --- /dev/null +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/MoneyStatisticsTest.java @@ -0,0 +1,28 @@ +package io.dockstore.metricsaggregator; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.math.RoundingMode; +import java.util.List; +import java.util.stream.Stream; +import javax.money.Monetary; +import javax.money.MonetaryRounding; +import javax.money.RoundingQueryBuilder; +import org.javamoney.moneta.Money; +import org.junit.jupiter.api.Test; + +class MoneyStatisticsTest { + @Test + void testStatistic() { + List fibonacci = Stream.of(0.0, 1.0, 1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 21.0, 34.0, 55.0, 89.0, 144.0) + .map(value -> Money.of(value, "USD")) + .toList(); + MoneyStatistics statistics = new MoneyStatistics(fibonacci); + assertEquals(Money.of(0, "USD"), statistics.getMinimum()); + assertEquals(Money.of(144, "USD"), statistics.getMaximum()); + MonetaryRounding rounding = Monetary.getRounding( + RoundingQueryBuilder.of().setScale(2).set(RoundingMode.HALF_UP).build()); + assertEquals(Money.of(28.92, "USD"), statistics.getAverage().with(rounding)); + assertEquals(13, statistics.getNumberOfDataPoints()); + } +} diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java index 9de43913..1b8a0f9c 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java @@ -23,6 +23,7 @@ import static io.dockstore.metricsaggregator.common.TestUtilities.CONFIG_FILE_PATH; import static io.dockstore.metricsaggregator.common.TestUtilities.ENDPOINT_OVERRIDE; import static io.dockstore.metricsaggregator.common.TestUtilities.createRunExecution; +import static io.dockstore.metricsaggregator.common.TestUtilities.createValidationExecution; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_RUNTIME_INVALID; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_SEMANTIC_INVALID; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.SUCCESSFUL; @@ -49,6 +50,7 @@ import io.dockstore.openapi.client.ApiClient; import io.dockstore.openapi.client.api.ExtendedGa4GhApi; import io.dockstore.openapi.client.api.WorkflowsApi; +import io.dockstore.openapi.client.model.Cost; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; @@ -62,7 +64,6 @@ import io.dropwizard.testing.DropwizardTestSupport; import io.dropwizard.testing.ResourceHelpers; import java.io.IOException; -import java.time.Instant; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -138,20 +139,12 @@ void testAggregateMetrics() { String versionId = version.getName(); // A successful run execution that ran for 5 minutes, requires 2 CPUs and 2 GBs of memory - List runExecutions = List.of(createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0)); + List runExecutions = List.of(createRunExecution(SUCCESSFUL, "PT5M", 2, 2.0, new Cost().value(2.00), "us-central1")); // A successful miniwdl validation final String validatorToolVersion1 = "1.0"; - ValidationExecution validationExecution1 = new ValidationExecution() - .validatorTool(MINIWDL) - .validatorToolVersion(validatorToolVersion1) - .isValid(true) - .dateExecuted(Instant.now().toString()); + ValidationExecution validationExecution1 = createValidationExecution(MINIWDL, validatorToolVersion1, true); final String validatorToolVersion2 = "2.0"; - ValidationExecution validationExecution2 = new ValidationExecution() - .validatorTool(WOMTOOL) - .validatorToolVersion(validatorToolVersion2) - .isValid(false) - .dateExecuted(Instant.now().toString()); + ValidationExecution validationExecution2 = createValidationExecution(WOMTOOL, validatorToolVersion2, false); // Submit metrics for two platforms extendedGa4GhApi.executionMetricsPost(new ExecutionsRequestBody().runExecutions(runExecutions).validationExecutions(List.of(validationExecution1)), platform1, id, versionId, ""); @@ -164,17 +157,19 @@ void testAggregateMetrics() { version = workflow.getWorkflowVersions().stream().filter(v -> "master".equals(v.getName())).findFirst().orElse(null); assertNotNull(version); assertEquals(expectedNumberOfPlatforms, version.getMetricsByPlatform().size(), "There should be metrics for two platforms"); + assertAggregatedMetricsForPlatform(platform1, version, validationExecution1); + assertAggregatedMetricsForPlatform(platform2, version, validationExecution2); + Metrics platform1Metrics = version.getMetricsByPlatform().get(platform1); assertNotNull(platform1Metrics); - compareAggregateMetricsWithPlatforms(platform2, version, validatorToolVersion1, validatorToolVersion2, platform1Metrics); ValidatorVersionInfo mostRecentValidationVersionInfo; ValidatorInfo validationInfo; // A failed run execution that ran for 1 second, requires 2 CPUs and 4.5 GBs of memory - runExecutions = List.of(createRunExecution(FAILED_RUNTIME_INVALID, "PT1S", 4, 4.5)); + runExecutions = List.of(createRunExecution(FAILED_RUNTIME_INVALID, "PT1S", 4, 4.5, new Cost().value(2.00), "us-central1")); // A failed miniwdl validation for the same validator version - List validationExecutions = List.of(new ValidationExecution().validatorTool(MINIWDL).validatorToolVersion("1.0").isValid(false).dateExecuted(Instant.now().toString())); + List validationExecutions = List.of(createValidationExecution(MINIWDL, "1.0", false)); ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().runExecutions(runExecutions).validationExecutions(validationExecutions); // Submit metrics for the same workflow version for platform 2 extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform1, id, versionId, ""); @@ -206,6 +201,12 @@ void testAggregateMetrics() { assertEquals(3.25, platform1Metrics.getMemory().getAverage()); assertNotNull(platform1Metrics.getMemory().getUnit()); + assertEquals(2, platform1Metrics.getCost().getNumberOfDataPointsForAverage()); + assertEquals(2, platform1Metrics.getCost().getMinimum()); + assertEquals(2, platform1Metrics.getCost().getMaximum()); + assertEquals(2, platform1Metrics.getCost().getAverage()); + assertNotNull(platform1Metrics.getCost().getUnit()); + assertEquals(2, platform1Metrics.getExecutionTime().getNumberOfDataPointsForAverage()); assertEquals(1, platform1Metrics.getExecutionTime().getMinimum()); assertEquals(300, platform1Metrics.getExecutionTime().getMaximum()); @@ -224,89 +225,66 @@ void testAggregateMetrics() { assertEquals(50d, validationInfo.getPassingRate()); assertEquals(2, validationInfo.getNumberOfRuns()); - testAggregatedMetrics(version, validatorToolVersion1, validatorToolVersion2, platform1Metrics); + testOverallAggregatedMetrics(version, validatorToolVersion1, validatorToolVersion2, platform1Metrics); } - private static void compareAggregateMetricsWithPlatforms(String platform2, WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) { - // Verify that the aggregated metrics are the same as the single execution for platform1 - assertEquals(1, platform1Metrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions()); - assertEquals(0, platform1Metrics.getExecutionStatusCount().getNumberOfFailedExecutions()); - assertEquals(1, platform1Metrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name())); - assertFalse(platform1Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name())); - assertFalse(platform1Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name())); - - assertEquals(1, platform1Metrics.getCpu().getNumberOfDataPointsForAverage()); - assertEquals(2, platform1Metrics.getCpu().getMinimum()); - assertEquals(2, platform1Metrics.getCpu().getMaximum()); - assertEquals(2, platform1Metrics.getCpu().getAverage()); - assertNull(platform1Metrics.getCpu().getUnit()); - - assertEquals(1, platform1Metrics.getMemory().getNumberOfDataPointsForAverage()); - assertEquals(2, platform1Metrics.getMemory().getMinimum()); - assertEquals(2, platform1Metrics.getMemory().getMaximum()); - assertEquals(2, platform1Metrics.getMemory().getAverage()); - assertNotNull(platform1Metrics.getMemory().getUnit()); - - assertEquals(1, platform1Metrics.getExecutionTime().getNumberOfDataPointsForAverage()); - assertEquals(300, platform1Metrics.getExecutionTime().getMinimum()); - assertEquals(300, platform1Metrics.getExecutionTime().getMaximum()); - assertEquals(300, platform1Metrics.getExecutionTime().getAverage()); - assertNotNull(platform1Metrics.getExecutionTime().getUnit()); - - assertEquals(1, platform1Metrics.getValidationStatus().getValidatorTools().size()); - ValidatorInfo validationInfo = platform1Metrics.getValidationStatus().getValidatorTools().get(MINIWDL.toString()); + private static void assertAggregatedMetricsForPlatform(String platform, WorkflowVersion version, ValidationExecution submittedValidationExecution) { + Metrics platformMetrics = version.getMetricsByPlatform().get(platform); + assertNotNull(platformMetrics); + + // Verify that the aggregated metrics are the same as the single execution for the platform + assertEquals(1, platformMetrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions()); + assertEquals(0, platformMetrics.getExecutionStatusCount().getNumberOfFailedExecutions()); + assertEquals(1, platformMetrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name())); + assertFalse(platformMetrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name())); + assertFalse(platformMetrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name())); + + assertEquals(1, platformMetrics.getCpu().getNumberOfDataPointsForAverage()); + assertEquals(2, platformMetrics.getCpu().getMinimum()); + assertEquals(2, platformMetrics.getCpu().getMaximum()); + assertEquals(2, platformMetrics.getCpu().getAverage()); + assertNull(platformMetrics.getCpu().getUnit()); + + assertEquals(1, platformMetrics.getMemory().getNumberOfDataPointsForAverage()); + assertEquals(2, platformMetrics.getMemory().getMinimum()); + assertEquals(2, platformMetrics.getMemory().getMaximum()); + assertEquals(2, platformMetrics.getMemory().getAverage()); + assertNotNull(platformMetrics.getMemory().getUnit()); + + assertEquals(1, platformMetrics.getCost().getNumberOfDataPointsForAverage()); + assertEquals(2, platformMetrics.getCost().getMinimum()); + assertEquals(2, platformMetrics.getCost().getMaximum()); + assertEquals(2, platformMetrics.getCost().getAverage()); + assertNotNull(platformMetrics.getCost().getUnit()); + + assertEquals(1, platformMetrics.getExecutionTime().getNumberOfDataPointsForAverage()); + assertEquals(300, platformMetrics.getExecutionTime().getMinimum()); + assertEquals(300, platformMetrics.getExecutionTime().getMaximum()); + assertEquals(300, platformMetrics.getExecutionTime().getAverage()); + assertNotNull(platformMetrics.getExecutionTime().getUnit()); + + assertEquals(1, platformMetrics.getValidationStatus().getValidatorTools().size()); + final String expectedValidatorTool = submittedValidationExecution.getValidatorTool().toString(); + ValidatorInfo validationInfo = platformMetrics.getValidationStatus().getValidatorTools().get(expectedValidatorTool); assertNotNull(validationInfo); assertNotNull(validationInfo.getMostRecentVersionName()); - ValidatorVersionInfo mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion1.equals(validationVersion.getName())).findFirst().get(); - assertTrue(mostRecentValidationVersionInfo.isIsValid(), "miniwdl validation should be valid"); - assertEquals(validatorToolVersion1, mostRecentValidationVersionInfo.getName()); - assertEquals(100d, mostRecentValidationVersionInfo.getPassingRate()); - assertEquals(1, mostRecentValidationVersionInfo.getNumberOfRuns()); - assertEquals(100d, validationInfo.getPassingRate()); - assertEquals(1, validationInfo.getNumberOfRuns()); - Metrics platform2Metrics = version.getMetricsByPlatform().get(platform2); - assertNotNull(platform2Metrics); - - // Verify that the aggregated metrics are the same as the single execution for platform2 - assertEquals(1, platform2Metrics.getExecutionStatusCount().getNumberOfSuccessfulExecutions()); - assertEquals(0, platform2Metrics.getExecutionStatusCount().getNumberOfFailedExecutions()); - assertEquals(1, platform2Metrics.getExecutionStatusCount().getCount().get(SUCCESSFUL.name())); - assertFalse(platform2Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_RUNTIME_INVALID.name())); - assertFalse(platform2Metrics.getExecutionStatusCount().getCount().containsKey(FAILED_SEMANTIC_INVALID.name())); - - assertEquals(1, platform2Metrics.getCpu().getNumberOfDataPointsForAverage()); - assertEquals(2, platform2Metrics.getCpu().getMinimum()); - assertEquals(2, platform2Metrics.getCpu().getMaximum()); - assertEquals(2, platform2Metrics.getCpu().getAverage()); - assertNull(platform2Metrics.getCpu().getUnit()); - - assertEquals(1, platform2Metrics.getMemory().getNumberOfDataPointsForAverage()); - assertEquals(2, platform2Metrics.getMemory().getMinimum()); - assertEquals(2, platform2Metrics.getMemory().getMaximum()); - assertEquals(2, platform2Metrics.getMemory().getAverage()); - assertNotNull(platform2Metrics.getMemory().getUnit()); - - assertEquals(1, platform2Metrics.getExecutionTime().getNumberOfDataPointsForAverage()); - assertEquals(300, platform2Metrics.getExecutionTime().getMinimum()); - assertEquals(300, platform2Metrics.getExecutionTime().getMaximum()); - assertEquals(300, platform2Metrics.getExecutionTime().getAverage()); - assertNotNull(platform2Metrics.getExecutionTime().getUnit()); - - assertEquals(1, platform2Metrics.getValidationStatus().getValidatorTools().size()); - validationInfo = platform2Metrics.getValidationStatus().getValidatorTools().get(WOMTOOL.toString()); - assertNotNull(validationInfo); - assertNotNull(validationInfo.getMostRecentVersionName()); - mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion2.equals(validationVersion.getName())).findFirst().get(); - assertFalse(mostRecentValidationVersionInfo.isIsValid(), "womtool validation should be invalid"); - assertEquals(validatorToolVersion2, mostRecentValidationVersionInfo.getName()); - assertEquals(0d, mostRecentValidationVersionInfo.getPassingRate()); + final String expectedMostRecentValidationVersionName = submittedValidationExecution.getValidatorToolVersion(); + ValidatorVersionInfo mostRecentValidationVersionInfo = validationInfo.getValidatorVersions().stream().filter(validationVersion -> expectedMostRecentValidationVersionName.equals(validationVersion.getName())).findFirst().get(); + assertEquals(submittedValidationExecution.isIsValid(), mostRecentValidationVersionInfo.isIsValid()); + assertEquals(expectedMostRecentValidationVersionName, mostRecentValidationVersionInfo.getName()); assertEquals(1, mostRecentValidationVersionInfo.getNumberOfRuns()); - assertEquals(0d, validationInfo.getPassingRate()); assertEquals(1, validationInfo.getNumberOfRuns()); + if (submittedValidationExecution.isIsValid()) { + assertEquals(100d, mostRecentValidationVersionInfo.getPassingRate()); + assertEquals(100d, validationInfo.getPassingRate()); + } else { + assertEquals(0d, mostRecentValidationVersionInfo.getPassingRate()); + assertEquals(0d, validationInfo.getPassingRate()); + } } - private static void testAggregatedMetrics(WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) { + private static void testOverallAggregatedMetrics(WorkflowVersion version, String validatorToolVersion1, String validatorToolVersion2, Metrics platform1Metrics) { ValidatorVersionInfo mostRecentValidationVersionInfo; ValidatorInfo validationInfo; // Verify that the metrics aggregated across ALL platforms are correct diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java index 73813452..1123e55a 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java @@ -19,9 +19,13 @@ import io.dockstore.metricsaggregator.MetricsAggregatorConfig; import io.dockstore.metricsaggregator.client.cli.MetricsAggregatorClient; +import io.dockstore.openapi.client.model.Cost; import io.dockstore.openapi.client.model.RunExecution; +import io.dockstore.openapi.client.model.ValidationExecution; +import io.dockstore.openapi.client.model.ValidationExecution.ValidatorToolEnum; import io.dropwizard.testing.ResourceHelpers; import java.io.File; +import java.time.Instant; import java.util.Optional; import org.apache.commons.configuration2.INIConfiguration; @@ -34,12 +38,25 @@ public final class TestUtilities { private TestUtilities() { } - public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB) { - return new RunExecution() + public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB, Cost cost, String region) { + RunExecution runExecution = new RunExecution() .executionStatus(executionStatus) .executionTime(executionTime) .cpuRequirements(cpuRequirements) - .memoryRequirementsGB(memoryRequirementsGB); + .memoryRequirementsGB(memoryRequirementsGB) + .cost(cost) + .region(region); + runExecution.setDateExecuted(Instant.now().toString()); + return runExecution; + } + + public static ValidationExecution createValidationExecution(ValidatorToolEnum validatorTool, String validatorToolVersion, boolean isValid) { + ValidationExecution validationExecution = new ValidationExecution() + .validatorTool(validatorTool) + .validatorToolVersion(validatorToolVersion) + .isValid(isValid); + validationExecution.setDateExecuted(Instant.now().toString()); + return validationExecution; } public static MetricsAggregatorConfig getMetricsConfig() { diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java index 624c2646..ca255c2d 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java @@ -1,5 +1,6 @@ package io.dockstore.metricsaggregator.helper; +import static io.dockstore.metricsaggregator.common.TestUtilities.createValidationExecution; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_RUNTIME_INVALID; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_SEMANTIC_INVALID; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.SUCCESSFUL; @@ -9,6 +10,8 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.dockstore.openapi.client.model.Cost; +import io.dockstore.openapi.client.model.CostMetric; import io.dockstore.openapi.client.model.CpuMetric; import io.dockstore.openapi.client.model.ExecutionStatusMetric; import io.dockstore.openapi.client.model.ExecutionTimeMetric; @@ -166,6 +169,42 @@ void testGetAggregatedMemory() { assertEquals(3, memoryMetric.get().getNumberOfDataPointsForAverage()); } + @Test + void testGetAggregatedCost() { + List executions = new ArrayList<>(); + Optional costMetric = AggregationHelper.getAggregatedCost(new ExecutionsRequestBody().runExecutions(executions)); + assertTrue(costMetric.isEmpty()); + + // Add an execution that doesn't have cost data + executions.add(new RunExecution().executionStatus(SUCCESSFUL)); + costMetric = AggregationHelper.getAggregatedCost(new ExecutionsRequestBody().runExecutions(executions)); + assertTrue(costMetric.isEmpty()); + + // Add an execution with cost data + Double costInUSD = 2.00; + executions.add(new RunExecution().executionStatus(SUCCESSFUL).cost(new Cost().value(costInUSD))); + costMetric = AggregationHelper.getAggregatedCost(new ExecutionsRequestBody().runExecutions(executions)); + assertTrue(costMetric.isPresent()); + assertEquals(costInUSD, costMetric.get().getMinimum()); + assertEquals(costInUSD, costMetric.get().getMaximum()); + assertEquals(costInUSD, costMetric.get().getAverage()); + assertEquals(1, costMetric.get().getNumberOfDataPointsForAverage()); + + // Aggregate submissions containing run executions and aggregated metrics + Metrics submittedAggregatedMetrics = new Metrics() + .cost(new CostMetric() + .minimum(2.00) + .maximum(6.00) + .average(4.00) + .numberOfDataPointsForAverage(2)); + costMetric = AggregationHelper.getAggregatedCost(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); + assertTrue(costMetric.isPresent()); + assertEquals(2.0, costMetric.get().getMinimum()); + assertEquals(6.0, costMetric.get().getMaximum()); + assertEquals(3.333333333333333, costMetric.get().getAverage()); + assertEquals(3, costMetric.get().getNumberOfDataPointsForAverage()); + } + @Test void testGetAggregatedValidationStatus() { List executions = new ArrayList<>(); @@ -175,11 +214,7 @@ void testGetAggregatedValidationStatus() { // Add an execution with validation data final ValidationExecution.ValidatorToolEnum validatorTool = ValidationExecution.ValidatorToolEnum.MINIWDL; final String validatorToolVersion1 = "1.0"; - executions.add(new ValidationExecution() - .validatorTool(validatorTool) - .validatorToolVersion(validatorToolVersion1) - .isValid(true) - .dateExecuted(Instant.now().toString())); + executions.add(createValidationExecution(validatorTool, validatorToolVersion1, true)); validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); ValidatorInfo validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); @@ -197,12 +232,7 @@ void testGetAggregatedValidationStatus() { // Add an execution that isn't valid for the same validator final String validatorToolVersion2 = "2.0"; - executions.add(new ValidationExecution() - .validatorTool(validatorTool) - .validatorToolVersion(validatorToolVersion2) - .isValid(false) - .dateExecuted(Instant.now().toString()) - .errorMessage("This is an error message")); + executions.add(createValidationExecution(validatorTool, validatorToolVersion2, false).errorMessage("This is an error message")); validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); @@ -218,11 +248,9 @@ void testGetAggregatedValidationStatus() { // Add an execution that is valid for the same validator String expectedDateExecuted = Instant.now().toString(); - executions.add(new ValidationExecution() - .validatorTool(validatorTool) - .validatorToolVersion(validatorToolVersion1) - .isValid(true) - .dateExecuted(expectedDateExecuted)); + ValidationExecution validationExecution = createValidationExecution(validatorTool, validatorToolVersion1, true); + validationExecution.setDateExecuted(expectedDateExecuted); + executions.add(validationExecution); validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); diff --git a/pom.xml b/pom.xml index 04b33f9e..af7e5bae 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ scm:git:git@github.com:dockstore/dockstore-support.git UTF-8 - 1.15.0-alpha.4 + 1.15.0-alpha.5 3.0.0-M5 2.22.2 false