Aggregate cost metric

THIRD-PARTY-LICENSES.txt

@@ -1,5 +1,5 @@
 
-Lists of 399 third-party dependencies.
+Lists of 402 third-party dependencies.
      (Apache License, Version 2.0) akka-actor (com.typesafe.akka:akka-actor_2.13:2.5.32 - https://akka.io/)
      (Apache License, Version 2.0) akka-protobuf (com.typesafe.akka:akka-protobuf_2.13:2.5.32 - https://akka.io/)
      (Apache License, Version 2.0) akka-slf4j (com.typesafe.akka:akka-slf4j_2.13:2.5.32 - https://akka.io/)
@@ -124,10 +124,10 @@ Lists of 399 third-party dependencies.
      (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.0 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.0 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-transport-httpclient5 (com.github.docker-java:docker-java-transport-httpclient5:3.3.0 - https://github.com/docker-java/docker-java)
-     (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-alpha.4 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-alpha.4 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-alpha.4 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-alpha.4 - no url defined)
+     (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-SNAPSHOT - no url defined)
      (Apache License 2.0) Dropwizard (io.dropwizard:dropwizard-core:4.0.0 - http://www.dropwizard.io/4.0.0/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-core)
      (Apache License 2.0) Dropwizard Asset Bundle (io.dropwizard:dropwizard-assets:4.0.0 - http://www.dropwizard.io/4.0.0/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-assets)
      (Apache License 2.0) Dropwizard Authentication (io.dropwizard:dropwizard-auth:4.0.0 - http://www.dropwizard.io/4.0.0/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-auth)
@@ -228,6 +228,7 @@ Lists of 399 third-party dependencies.
      (The Apache Software License, Version 2.0) java-diff-utils (io.github.java-diff-utils:java-diff-utils:4.12 - https://github.com/java-diff-utils/java-diff-utils/java-diff-utils)
      (CDDL/GPLv2+CE) JavaBeans Activation Framework (com.sun.activation:javax.activation:1.2.0 - http://java.net/all/javax.activation/)
      (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.29.2-GA - http://www.javassist.org/)
+     (CDDL + GPLv2 with classpath exception) javax.annotation API (javax.annotation:javax.annotation-api:1.3.2 - http://jcp.org/en/jsr/detail?id=250)
      (Eclipse Distribution License - v 1.0) JAXB Core (org.glassfish.jaxb:jaxb-core:3.0.2 - https://eclipse-ee4j.github.io/jaxb-ri/)
      (Eclipse Distribution License - v 1.0) JAXB Runtime (org.glassfish.jaxb:jaxb-runtime:3.0.2 - https://eclipse-ee4j.github.io/jaxb-ri/)
      (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.3.Final - http://www.jboss.org)
@@ -278,6 +279,7 @@ Lists of 399 third-party dependencies.
      (Public Domain) JSON in Java (org.json:json:20230227 - https://github.com/douglascrockford/JSON-java)
      (Revised BSD License) JSONLD Java :: Core (com.github.jsonld-java:jsonld-java:0.8.3 - http://github.com/jsonld-java/jsonld-java/jsonld-java/)
      (The MIT License) jsoup Java HTML Parser (org.jsoup:jsoup:1.10.2 - https://jsoup.org/)
+     (Apache License, Version 2.0) JSR 354 (Money and Currency API) (javax.money:money-api:1.1 - https://javamoney.github.io/)
      (MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:2.0.7 - http://www.slf4j.org)
      (Eclipse Public License 1.0) JUnit (junit:junit:4.13.2 - http://junit.org)
      (Eclipse Public License v2.0) JUnit Jupiter (Aggregator) (org.junit.jupiter:junit-jupiter:5.9.2 - https://junit.org/junit5/)
@@ -318,6 +320,7 @@ Lists of 399 third-party dependencies.
      (Eclipse Distribution License - v 1.0) MIME streaming extension (org.jvnet.mimepull:mimepull:1.9.15 - https://github.com/eclipse-ee4j/metro-mimepull)
      (The MIT License) mockito-core (org.mockito:mockito-core:3.12.4 - https://github.com/mockito/mockito)
      (The MIT License) mockito-inline (org.mockito:mockito-inline:3.12.4 - https://github.com/mockito/mockito)
+     (Apache 2 License) Moneta Core (org.javamoney.moneta:moneta-core:1.4.2 - http://javamoney.org)
      (MIT license) mouse (org.typelevel:mouse_2.13:1.0.10 - https://typelevel.org/mouse)
      (Apache License, Version 2.0) Netty Reactive Streams HTTP support (com.typesafe.netty:netty-reactive-streams-http:2.0.5 - https://github.com/playframework/netty-reactive-streams/netty-reactive-streams-http)
      (Apache License, Version 2.0) Netty Reactive Streams Implementation (com.typesafe.netty:netty-reactive-streams:2.0.5 - https://github.com/playframework/netty-reactive-streams/netty-reactive-streams)
@@ -340,7 +343,7 @@ Lists of 399 third-party dependencies.
      (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis)
      (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/)
      (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/)
-     (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-alpha.4 - no url defined)
+     (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-SNAPSHOT - no url defined)
      (The Apache License, Version 2.0) OpenCensus (io.opencensus:opencensus-api:0.31.0 - https://github.com/census-instrumentation/opencensus-java)
      (Apache 2) opencsv (com.opencsv:opencsv:5.7.1 - http://opencsv.sf.net)
      (MIT License) ORCID - Model (org.orcid:orcid-model-jakarta:3.3.0 - https://github.com/ORCID/orcid-model)
@@ -378,7 +381,7 @@ Lists of 399 third-party dependencies.
      (Apache License 2.0) swagger-core-jakarta (io.swagger.core.v3:swagger-core-jakarta:2.2.9 - https://github.com/swagger-api/swagger-core/modules/swagger-core-jakarta)
      (Apache License 2.0) swagger-integration-jakarta (io.swagger.core.v3:swagger-integration-jakarta:2.2.9 - https://github.com/swagger-api/swagger-core/modules/swagger-integration-jakarta)
      (Apache Software License, Version 2.0) swagger-java-bitbucket-client (io.dockstore:swagger-java-bitbucket-client:2.0.3 - no url defined)
-     (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-alpha.4 - no url defined)
+     (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-SNAPSHOT - no url defined)
      (Apache Software License, Version 2.0) swagger-java-discourse-client (io.dockstore:swagger-java-discourse-client:2.0.1 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-quay-client (io.dockstore:swagger-java-quay-client:2.0.2 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-sam-client (io.dockstore:swagger-java-sam-client:2.0.2 - no url defined)

metricsaggregator/pom.xml

@@ -131,6 +131,14 @@
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>s3</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.javamoney.moneta</groupId>
+            <artifactId>moneta-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>javax.money</groupId>
+            <artifactId>money-api</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
@@ -263,6 +271,8 @@
                                 <usedDependency>org.slf4j:slf4j-api</usedDependency>
                                 <usedDependency>software.amazon.awssdk:s3</usedDependency>
                                 <usedDependency>org.glassfish.jersey.inject:jersey-hk2</usedDependency>
+                                <usedDependency>javax.money:money-api</usedDependency>
+                                <usedDependency>org.javamoney.moneta:moneta-core</usedDependency>
                             </usedDependencies>
                         </configuration>
                     </execution>

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/DoubleStatistics.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2023 OICR and UCSC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *           http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package io.dockstore.metricsaggregator;
+
+import java.util.List;
+
+/**
+ * Record that contains statistical information obtained from a list of Doubles.
+ */
+public class DoubleStatistics extends Statistics<Double> {
+    public DoubleStatistics() {
+        super();
+    }
+
+    /**
+     * Constructor that calculates statistical information from the provided list of data points.
+     * @param dataPoints List of Doubles
+     */
+    public DoubleStatistics(List<Double> dataPoints) {
+        super(dataPoints);
+    }
+
+    public DoubleStatistics(Double minimum, Double maximum, Double average, int numberOfDataPoints) {
+        super(minimum, maximum, average, numberOfDataPoints);
+    }
+
+    /**
+     * Constructor used to create a Statistics object that can be used to calculate weighted averages for non-Statistics objects.
+     * A placeholder value is set for the min and maximum fields
+     * @param average
+     * @param numberOfDataPoints
+     */
+    public DoubleStatistics(double average, int numberOfDataPoints) {
+        super(0d, 0d, average, numberOfDataPoints);
+    }
+
+    /**
+     * Create a new Statistics object from a list of statistics by aggregating the list of statistics
+     * @param statistics
+     * @return
+     */
+    public static DoubleStatistics createFromStatistics(List<DoubleStatistics> statistics) {
+        if (statistics.size() == 1) {
+            return statistics.get(0);
+        }
+
+        DoubleStatistics newStatistics = new DoubleStatistics();
+        newStatistics.setAverage(statistics);
+        newStatistics.setMinimum(statistics);
+        newStatistics.setMaximum(statistics);
+        newStatistics.setNumberOfDataPoints(statistics);
+        return newStatistics;
+    }
+
+    /**
+     * Get the lowest value from the list of data points.
+     * @param dataPoints
+     * @return
+     */
+    @Override
+    public Double calculateMinimum(List<Double> dataPoints) {
+        return dataPoints.stream().mapToDouble(d -> d).min().getAsDouble();
+    }
+
+    /**
+     * Get the highest value from the list of data points.
+     * @param dataPoints
+     * @return
+     */
+    @Override
+    public Double calculateMaximum(List<Double> dataPoints) {
+        return dataPoints.stream().mapToDouble(d -> d).max().getAsDouble();
+    }
+
+    /**
+     * Calculate the average from the list of data points.
+     * @param dataPoints
+     * @return
+     */
+    @Override
+    public Double calculateAverage(List<Double> dataPoints) {
+        return dataPoints.stream().mapToDouble(d -> d).average().getAsDouble();
+    }
+
+    /**
+     * Calculate a weighted average
+     */
+    @Override
+    public Double calculateWeightedAverage(List<? extends Statistics<Double>> statistics) {
+        int totalNumberOfDataPoints = getTotalNumberOfDataPoints(statistics);
+        return statistics.stream()
+                .map(stat -> {
+                    double weight = (double)stat.getNumberOfDataPoints() / (double)totalNumberOfDataPoints;
+                    return stat.getAverage() * weight;
+                })
+                .mapToDouble(Double::doubleValue)
+                .sum();
+    }
+}

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java

@@ -73,6 +73,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) {
             return;
         }
 
+        System.out.println("Aggregating metrics...");
         for (S3DirectoryInfo directoryInfo : metricsDirectories) {
             String toolId = directoryInfo.toolId();
             String versionName = directoryInfo.versionId();
@@ -118,6 +119,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) {
                 }
             }
         }
+        System.out.println("Completed aggregating metrics");
     }
 
     /**

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MoneyStatistics.java

@@ -0,0 +1,90 @@
+package io.dockstore.metricsaggregator;
+
+import java.util.List;
+import org.javamoney.moneta.Money;
+
+/**
+ * Calculates money statistics in USD using the Java Money library to preserve accuracy.
+ */
+public class MoneyStatistics extends Statistics<Money> {
+    private static String currency = "USD";
+
+    private MoneyStatistics() {
+        super();
+    }
+
+    public MoneyStatistics(List<Money> dataPoints) {
+        super(dataPoints);
+    }
+
+    public MoneyStatistics(Money minimum, Money maximum, Money average, int numberOfDataPoints) {
+        super(minimum, maximum, average, numberOfDataPoints);
+    }
+
+    /**
+     * Create a new Statistics object from a list of statistics by aggregating the list of statistics
+     * @param statistics
+     * @return
+     */
+    public static MoneyStatistics createFromStatistics(List<MoneyStatistics> statistics) {
+        if (statistics.size() == 1) {
+            return statistics.get(0);
+        }
+
+        MoneyStatistics newStatistics = new MoneyStatistics();
+        newStatistics.setAverage(statistics);
+        newStatistics.setMinimum(statistics);
+        newStatistics.setMaximum(statistics);
+        newStatistics.setNumberOfDataPoints(statistics);
+        return newStatistics;
+    }
+
+    /**
+     * Get the lowest value from the list of data points.
+     * @param dataPoints
+     * @return
+     */
+    @Override
+    public Money calculateMinimum(List<Money> dataPoints) {
+        return dataPoints.stream()
+                .min(Money::compareTo)
+                .get();
+    }
+
+    /**
+     * Get the highest value from the list of data points.
+     * @param dataPoints
+     * @return
+     */
+    @Override
+    public Money calculateMaximum(List<Money> dataPoints) {
+        return dataPoints.stream()
+                .max(Money::compareTo)
+                .get();
+    }
+
+    /**
+     * Calculate the average from the list of data points.
+     * @param dataPoints
+     * @return
+     */
+    @Override
+    public Money calculateAverage(List<Money> dataPoints) {
+        Money sum = dataPoints.stream().reduce(Money.of(0, currency), Money::add);
+        return sum.divide(dataPoints.size());
+    }
+
+    /**
+     * Calculate a weighted average
+     */
+    @Override
+    public Money calculateWeightedAverage(List<? extends Statistics<Money>> statistics) {
+        int totalNumberOfDataPoints = getTotalNumberOfDataPoints(statistics);
+        return statistics.stream()
+                .map(stat -> {
+                    double weight = (double)stat.getNumberOfDataPoints() / (double)totalNumberOfDataPoints;
+                    return stat.getAverage().multiply(weight);
+                })
+                .reduce(Money.of(0, currency), Money::add);
+    }
+}