Use getEntryVersionsToAggregate endpoint to get entries to aggregate (#…

…500)

https://ucsc-cgl.atlassian.net/browse/SEAB-6467
* Use new endpoint to get entries to aggregate
* Update webservice tag
* Log exception

THIRD-PARTY-LICENSES.txt

@@ -129,10 +129,10 @@ Lists of 416 third-party dependencies.
      (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.6 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.6 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-transport-httpclient5 (com.github.docker-java:docker-java-transport-httpclient5:3.3.6 - https://github.com/docker-java/docker-java)
-     (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.16.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.16.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.16.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.16.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.16.0-alpha.16 - no url defined)
+     (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.16.0-alpha.16 - no url defined)
+     (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.16.0-alpha.16 - no url defined)
+     (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.16.0-alpha.16 - no url defined)
      (Apache License 2.0) Dropwizard (io.dropwizard:dropwizard-core:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-core)
      (Apache License 2.0) Dropwizard Asset Bundle (io.dropwizard:dropwizard-assets:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-assets)
      (Apache License 2.0) Dropwizard Authentication (io.dropwizard:dropwizard-auth:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-auth)
@@ -352,7 +352,7 @@ Lists of 416 third-party dependencies.
      (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis)
      (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/)
      (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/)
-     (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.16.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.16.0-alpha.16 - no url defined)
      (The Apache License, Version 2.0) OpenCensus (io.opencensus:opencensus-api:0.31.0 - https://github.com/census-instrumentation/opencensus-java)
      (Apache 2) opencsv (com.opencsv:opencsv:5.7.1 - http://opencsv.sf.net)
      (Apache 2.0) optics (io.circe:circe-optics_2.13:0.14.1 - https://github.com/circe/circe-optics)
@@ -394,7 +394,7 @@ Lists of 416 third-party dependencies.
      (Apache License 2.0) swagger-core-jakarta (io.swagger.core.v3:swagger-core-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-core-jakarta)
      (Apache License 2.0) swagger-integration-jakarta (io.swagger.core.v3:swagger-integration-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-integration-jakarta)
      (Apache Software License, Version 2.0) swagger-java-bitbucket-client (io.dockstore:swagger-java-bitbucket-client:2.0.3 - no url defined)
-     (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.16.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.16.0-alpha.16 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-discourse-client (io.dockstore:swagger-java-discourse-client:2.0.1 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-quay-client (io.dockstore:swagger-java-quay-client:2.0.2 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-sam-client (io.dockstore:swagger-java-sam-client:2.0.2 - no url defined)

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorAthenaClient.java

@@ -48,8 +48,8 @@ public class MetricsAggregatorAthenaClient {
     private final MetadataApi metadataApi;
 
     private final AtomicInteger numberOfDirectoriesProcessed = new AtomicInteger(0);
-    private final AtomicInteger numberOfMetricsSubmitted = new AtomicInteger(0);
-    private final AtomicInteger numberOfMetricsSkipped = new AtomicInteger(0);
+    private final AtomicInteger numberOfVersionsSubmitted = new AtomicInteger(0);
+    private final AtomicInteger numberOfVersionsSkipped = new AtomicInteger(0);
 
     public MetricsAggregatorAthenaClient(MetricsAggregatorConfig config) {
         this.metricsBucketName = config.getS3Config().bucket();
@@ -79,26 +79,25 @@ public void aggregateMetrics(List<S3DirectoryInfo> s3DirectoriesToAggregate, Ext
             Map<String, Metrics> platformToMetrics = getAggregatedMetricsForPlatforms(s3DirectoryInfo);
             if (platformToMetrics.isEmpty()) {
                 LOG.error("No metrics were aggregated for tool ID: {}, version {}", s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId());
-                numberOfMetricsSkipped.incrementAndGet();
+                numberOfVersionsSkipped.incrementAndGet();
             }
-            platformToMetrics.forEach((platform, metrics) -> {
-                if (!skipPostingToDockstore) {
-                    try {
-                        extendedGa4GhApi.aggregatedMetricsPut(metrics, platform, s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId());
-                        LOG.info("Posted aggregated metrics to Dockstore for tool ID: {}, version {}, platform: {}",
-                                s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId(), platform);
-                        numberOfMetricsSubmitted.incrementAndGet();
-                    } catch (ApiException exception) {
-                        // Log error and continue processing for other platforms
-                        LOG.error("Could not post aggregated metrics to Dockstore for tool ID: {}, version {}, platform: {}", s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId(), platform);
-                        numberOfMetricsSkipped.incrementAndGet();
-                    }
+            if (!skipPostingToDockstore) {
+                try {
+                    extendedGa4GhApi.aggregatedMetricsPut(platformToMetrics, s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId());
+                    LOG.info("Posted aggregated metrics to Dockstore for tool ID: {}, version {}, platform(s): {}",
+                            s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId(), platformToMetrics.keySet());
+                    numberOfVersionsSubmitted.incrementAndGet();
+                } catch (ApiException exception) {
+                    // Log error and continue processing for other platforms
+                    LOG.error("Could not post aggregated metrics to Dockstore for tool ID: {}, version {}, platform(s): {}", s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId(), platformToMetrics.keySet(), exception);
+                    numberOfVersionsSkipped.incrementAndGet();
                 }
-            });
+            }
             numberOfDirectoriesProcessed.incrementAndGet();
             LOG.info("Processed {} directories", numberOfDirectoriesProcessed);
         });
-        LOG.info("Completed aggregating metrics. Processed {} directories, submitted {} platform metrics, and skipped {} platform metrics", numberOfDirectoriesProcessed, numberOfMetricsSubmitted, numberOfMetricsSkipped);
+        LOG.info("Completed aggregating metrics. Processed {} directories, submitted metrics for {} versions, and skipped metrics for {} versions", numberOfDirectoriesProcessed,
+                numberOfVersionsSubmitted, numberOfVersionsSkipped);
     }
 
     /**

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java

@@ -54,8 +54,8 @@ public class MetricsAggregatorS3Client {
     private static final Logger LOG = LoggerFactory.getLogger(MetricsAggregatorS3Client.class);
     private static final Gson GSON = new Gson();
     private final AtomicInteger numberOfDirectoriesProcessed = new AtomicInteger(0);
-    private final AtomicInteger numberOfMetricsSubmitted = new AtomicInteger(0);
-    private final AtomicInteger numberOfMetricsSkipped = new AtomicInteger(0);
+    private final AtomicInteger numberOfVersionsSubmitted = new AtomicInteger(0);
+    private final AtomicInteger numberOfVersionsSkipped = new AtomicInteger(0);
 
     private final String bucketName;
 
@@ -76,69 +76,67 @@ public MetricsAggregatorS3Client(String bucketName, String s3EndpointOverride) t
 
     public void aggregateMetrics(List<S3DirectoryInfo> s3DirectoriesToAggregate, ExtendedGa4GhApi extendedGa4GhApi, boolean skipDockstore) {
         s3DirectoriesToAggregate.stream().parallel().forEach(directoryInfo -> aggregateMetricsForDirectory(directoryInfo, extendedGa4GhApi, skipDockstore));
-        LOG.info("Completed aggregating metrics. Processed {} directories, submitted {} platform metrics, and skipped {} platform metrics", numberOfDirectoriesProcessed, numberOfMetricsSubmitted, numberOfMetricsSkipped);
+        LOG.info("Completed aggregating metrics. Processed {} directories, submitted {} platform metrics, and skipped {} platform metrics", numberOfDirectoriesProcessed,
+                numberOfVersionsSubmitted,
+                numberOfVersionsSkipped);
     }
 
     private void aggregateMetricsForDirectory(S3DirectoryInfo directoryInfo, ExtendedGa4GhApi extendedGa4GhApi, boolean skipDockstore) {
         LOG.info("Processing directory {}", directoryInfo);
         String toolId = directoryInfo.toolId();
         String versionName = directoryInfo.versionId();
         List<String> platforms = directoryInfo.platforms();
-        String platformsString = String.join(", ", platforms);
         String versionS3KeyPrefix = directoryInfo.versionS3KeyPrefix();
 
         // Collect metrics for each platform, so we can calculate metrics across all platforms
-        List<Metrics> allMetrics = new ArrayList<>();
+        Map<String, Metrics> platformToMetrics = new HashMap<>();
         for (String platform : platforms) {
             ExecutionsRequestBody allSubmissions;
             try {
                 allSubmissions = getExecutions(toolId, versionName, platform);
             } catch (Exception e) {
                 LOG.error("Error aggregating metrics: Could not get all executions from directory {}", versionS3KeyPrefix, e);
-                numberOfMetricsSkipped.incrementAndGet();
+                numberOfVersionsSkipped.incrementAndGet();
                 continue; // Continue aggregating metrics for other directories
             }
 
             try {
                 Optional<Metrics> aggregatedPlatformMetric = getAggregatedMetrics(allSubmissions);
                 if (aggregatedPlatformMetric.isPresent()) {
-                    if (!skipDockstore) {
-                        extendedGa4GhApi.aggregatedMetricsPut(aggregatedPlatformMetric.get(), platform, toolId, versionName);
-                    }
                     LOG.info("Aggregated metrics for tool ID {}, version {}, platform {} from directory {}", toolId, versionName, platform,
                             versionS3KeyPrefix);
-                    allMetrics.add(aggregatedPlatformMetric.get());
-                    numberOfMetricsSubmitted.incrementAndGet();
+                    platformToMetrics.put(platform, aggregatedPlatformMetric.get());
                 } else {
                     LOG.error("Error aggregating metrics for tool ID {}, version {}, platform {} from directory {}", toolId, versionName, platform, versionS3KeyPrefix);
                 }
             } catch (Exception e) {
                 LOG.error("Error aggregating metrics: Could not put all executions from directory {}", versionS3KeyPrefix, e);
-                numberOfMetricsSkipped.incrementAndGet();
+                numberOfVersionsSkipped.incrementAndGet();
                 // Continue aggregating metrics for other platforms
             }
         }
 
-        if (!allMetrics.isEmpty()) {
+        if (!platformToMetrics.isEmpty()) {
             // Calculate metrics across all platforms by aggregating the aggregated metrics from each platform
             try {
-                getAggregatedMetrics(allMetrics).ifPresent(metrics -> {
+                getAggregatedMetrics(platformToMetrics.values().stream().toList()).ifPresent(metrics -> {
+                    platformToMetrics.put(Partner.ALL.name(), metrics);
                     if (!skipDockstore) {
-                        extendedGa4GhApi.aggregatedMetricsPut(metrics, Partner.ALL.name(), toolId, versionName);
+                        extendedGa4GhApi.aggregatedMetricsPut(platformToMetrics, toolId, versionName);
                     }
                     LOG.info("Aggregated metrics across all platforms ({}) for tool ID {}, version {} from directory {}",
-                            platformsString, toolId, versionName, versionS3KeyPrefix);
-                    allMetrics.add(metrics);
-                    numberOfMetricsSubmitted.incrementAndGet();
+                            platformToMetrics.keySet(), toolId, versionName, versionS3KeyPrefix);
+
+                    numberOfVersionsSubmitted.incrementAndGet();
                 });
             } catch (Exception e) {
-                LOG.error("Error aggregating metrics across all platforms ({}) for tool ID {}, version {} from directory {}", platformsString, toolId, versionName, versionS3KeyPrefix, e);
-                numberOfMetricsSkipped.incrementAndGet();
+                LOG.error("Error aggregating metrics across all platforms ({}) for tool ID {}, version {} from directory {}", platformToMetrics.keySet(), toolId, versionName, versionS3KeyPrefix, e);
+                numberOfVersionsSkipped.incrementAndGet();
                 // Continue aggregating metrics for other directories
             }
         } else {
             LOG.error("Error aggregating metrics for directory {}: no platform metrics aggregated", versionS3KeyPrefix);
-            numberOfMetricsSkipped.incrementAndGet();
+            numberOfVersionsSkipped.incrementAndGet();
         }
         numberOfDirectoriesProcessed.incrementAndGet();
         LOG.info("Processed {} directories", numberOfDirectoriesProcessed);
@@ -292,6 +290,12 @@ public List<S3DirectoryInfo> getDirectoriesForTrsId(String trsId) {
         return getDirectories(s3KeyPrefix);
     }
 
+    public List<S3DirectoryInfo> getDirectoriesForTrsIdVersion(String trsId, String versionName) {
+        final String s3KeyPrefix = S3ClientHelper.convertToolIdToPartialKey(trsId) + "/" + versionName;
+        LOG.info("Getting directories for TRS ID {} and version {} with S3 key prefix {}", trsId, versionName, s3KeyPrefix);
+        return getDirectories(s3KeyPrefix);
+    }
+
     public record S3DirectoryInfo(String toolId, String versionId, List<String> platforms, String versionS3KeyPrefix, AthenaTablePartition athenaTablePartition) {
     }
 }

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java

@@ -35,6 +35,7 @@
 import io.dockstore.metricsaggregator.client.cli.CommandLineArgs.SubmitValidationData;
 import io.dockstore.openapi.client.ApiClient;
 import io.dockstore.openapi.client.api.ExtendedGa4GhApi;
+import io.dockstore.openapi.client.model.EntryLiteAndVersionName;
 import io.dockstore.openapi.client.model.ExecutionsRequestBody;
 import io.dockstore.openapi.client.model.ValidationExecution;
 import io.dockstore.openapi.client.model.ValidationExecution.ValidatorToolEnum;
@@ -165,8 +166,12 @@ private void aggregateMetrics(AggregateMetricsCommand aggregateMetricsCommand, M
         final Instant getDirectoriesStartTime = Instant.now();
         List<S3DirectoryInfo> s3DirectoriesToAggregate;
         if (trsIdsToAggregate == null || trsIdsToAggregate.isEmpty()) {
-            LOG.info("Aggregating metrics for all entries");
-            s3DirectoriesToAggregate = metricsAggregatorS3Client.getDirectories(); // Aggregate all directories
+            LOG.info("Aggregating metrics for all entries that have new executions to aggregate");
+            List<EntryLiteAndVersionName> entryVersionsToAggregate = extendedGa4GhApi.getEntryVersionsToAggregate();
+            s3DirectoriesToAggregate = entryVersionsToAggregate.stream()
+                    .map(entryVersion -> metricsAggregatorS3Client.getDirectoriesForTrsIdVersion(entryVersion.getEntryLite().getTrsId(), entryVersion.getVersionName()))
+                    .flatMap(Collection::stream)
+                    .toList();
         } else {
             LOG.info("Aggregating metrics for TRS IDs: {}", trsIdsToAggregate);
             s3DirectoriesToAggregate = trsIdsToAggregate.stream()

pom.xml

@@ -38,7 +38,7 @@
 
         <github.url>scm:git:git@github.com:dockstore/dockstore-support.git</github.url>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <dockstore-core.version>1.16.0-alpha.13</dockstore-core.version>
+        <dockstore-core.version>1.16.0-alpha.16</dockstore-core.version>
         <maven-surefire.version>3.0.0-M5</maven-surefire.version>
         <maven-failsafe.version>2.22.2</maven-failsafe.version>
         <skipTests>false</skipTests>