diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 9033be28..9b4ac2f1 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -129,10 +129,10 @@ Lists of 416 third-party dependencies. (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.6 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.6 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport-httpclient5 (com.github.docker-java:docker-java-transport-httpclient5:3.3.6 - https://github.com/docker-java/docker-java) - (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.16.0-alpha.13 - no url defined) - (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.16.0-alpha.13 - no url defined) - (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.16.0-alpha.13 - no url defined) - (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.16.0-alpha.13 - no url defined) + (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.16.0-alpha.16 - no url defined) + (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.16.0-alpha.16 - no url defined) + (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.16.0-alpha.16 - no url defined) + (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.16.0-alpha.16 - no url defined) (Apache License 2.0) Dropwizard (io.dropwizard:dropwizard-core:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-core) (Apache License 2.0) Dropwizard Asset Bundle (io.dropwizard:dropwizard-assets:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-assets) (Apache License 2.0) Dropwizard Authentication (io.dropwizard:dropwizard-auth:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-auth) @@ -352,7 +352,7 @@ Lists of 416 third-party dependencies. (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis) (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/) (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/) - (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.16.0-alpha.13 - no url defined) + (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.16.0-alpha.16 - no url defined) (The Apache License, Version 2.0) OpenCensus (io.opencensus:opencensus-api:0.31.0 - https://github.com/census-instrumentation/opencensus-java) (Apache 2) opencsv (com.opencsv:opencsv:5.7.1 - http://opencsv.sf.net) (Apache 2.0) optics (io.circe:circe-optics_2.13:0.14.1 - https://github.com/circe/circe-optics) @@ -394,7 +394,7 @@ Lists of 416 third-party dependencies. (Apache License 2.0) swagger-core-jakarta (io.swagger.core.v3:swagger-core-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-core-jakarta) (Apache License 2.0) swagger-integration-jakarta (io.swagger.core.v3:swagger-integration-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-integration-jakarta) (Apache Software License, Version 2.0) swagger-java-bitbucket-client (io.dockstore:swagger-java-bitbucket-client:2.0.3 - no url defined) - (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.16.0-alpha.13 - no url defined) + (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.16.0-alpha.16 - no url defined) (Apache Software License, Version 2.0) swagger-java-discourse-client (io.dockstore:swagger-java-discourse-client:2.0.1 - no url defined) (Apache Software License, Version 2.0) swagger-java-quay-client (io.dockstore:swagger-java-quay-client:2.0.2 - no url defined) (Apache Software License, Version 2.0) swagger-java-sam-client (io.dockstore:swagger-java-sam-client:2.0.2 - no url defined) diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorAthenaClient.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorAthenaClient.java index fd8ca49b..8ad2cccc 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorAthenaClient.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorAthenaClient.java @@ -48,8 +48,8 @@ public class MetricsAggregatorAthenaClient { private final MetadataApi metadataApi; private final AtomicInteger numberOfDirectoriesProcessed = new AtomicInteger(0); - private final AtomicInteger numberOfMetricsSubmitted = new AtomicInteger(0); - private final AtomicInteger numberOfMetricsSkipped = new AtomicInteger(0); + private final AtomicInteger numberOfVersionsSubmitted = new AtomicInteger(0); + private final AtomicInteger numberOfVersionsSkipped = new AtomicInteger(0); public MetricsAggregatorAthenaClient(MetricsAggregatorConfig config) { this.metricsBucketName = config.getS3Config().bucket(); @@ -79,26 +79,25 @@ public void aggregateMetrics(List s3DirectoriesToAggregate, Ext Map platformToMetrics = getAggregatedMetricsForPlatforms(s3DirectoryInfo); if (platformToMetrics.isEmpty()) { LOG.error("No metrics were aggregated for tool ID: {}, version {}", s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId()); - numberOfMetricsSkipped.incrementAndGet(); + numberOfVersionsSkipped.incrementAndGet(); } - platformToMetrics.forEach((platform, metrics) -> { - if (!skipPostingToDockstore) { - try { - extendedGa4GhApi.aggregatedMetricsPut(metrics, platform, s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId()); - LOG.info("Posted aggregated metrics to Dockstore for tool ID: {}, version {}, platform: {}", - s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId(), platform); - numberOfMetricsSubmitted.incrementAndGet(); - } catch (ApiException exception) { - // Log error and continue processing for other platforms - LOG.error("Could not post aggregated metrics to Dockstore for tool ID: {}, version {}, platform: {}", s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId(), platform); - numberOfMetricsSkipped.incrementAndGet(); - } + if (!skipPostingToDockstore) { + try { + extendedGa4GhApi.aggregatedMetricsPut(platformToMetrics, s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId()); + LOG.info("Posted aggregated metrics to Dockstore for tool ID: {}, version {}, platform(s): {}", + s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId(), platformToMetrics.keySet()); + numberOfVersionsSubmitted.incrementAndGet(); + } catch (ApiException exception) { + // Log error and continue processing for other platforms + LOG.error("Could not post aggregated metrics to Dockstore for tool ID: {}, version {}, platform(s): {}", s3DirectoryInfo.toolId(), s3DirectoryInfo.versionId(), platformToMetrics.keySet(), exception); + numberOfVersionsSkipped.incrementAndGet(); } - }); + } numberOfDirectoriesProcessed.incrementAndGet(); LOG.info("Processed {} directories", numberOfDirectoriesProcessed); }); - LOG.info("Completed aggregating metrics. Processed {} directories, submitted {} platform metrics, and skipped {} platform metrics", numberOfDirectoriesProcessed, numberOfMetricsSubmitted, numberOfMetricsSkipped); + LOG.info("Completed aggregating metrics. Processed {} directories, submitted metrics for {} versions, and skipped metrics for {} versions", numberOfDirectoriesProcessed, + numberOfVersionsSubmitted, numberOfVersionsSkipped); } /** diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java index 6fc2fc45..370aec63 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java @@ -54,8 +54,8 @@ public class MetricsAggregatorS3Client { private static final Logger LOG = LoggerFactory.getLogger(MetricsAggregatorS3Client.class); private static final Gson GSON = new Gson(); private final AtomicInteger numberOfDirectoriesProcessed = new AtomicInteger(0); - private final AtomicInteger numberOfMetricsSubmitted = new AtomicInteger(0); - private final AtomicInteger numberOfMetricsSkipped = new AtomicInteger(0); + private final AtomicInteger numberOfVersionsSubmitted = new AtomicInteger(0); + private final AtomicInteger numberOfVersionsSkipped = new AtomicInteger(0); private final String bucketName; @@ -76,7 +76,9 @@ public MetricsAggregatorS3Client(String bucketName, String s3EndpointOverride) t public void aggregateMetrics(List s3DirectoriesToAggregate, ExtendedGa4GhApi extendedGa4GhApi, boolean skipDockstore) { s3DirectoriesToAggregate.stream().parallel().forEach(directoryInfo -> aggregateMetricsForDirectory(directoryInfo, extendedGa4GhApi, skipDockstore)); - LOG.info("Completed aggregating metrics. Processed {} directories, submitted {} platform metrics, and skipped {} platform metrics", numberOfDirectoriesProcessed, numberOfMetricsSubmitted, numberOfMetricsSkipped); + LOG.info("Completed aggregating metrics. Processed {} directories, submitted {} platform metrics, and skipped {} platform metrics", numberOfDirectoriesProcessed, + numberOfVersionsSubmitted, + numberOfVersionsSkipped); } private void aggregateMetricsForDirectory(S3DirectoryInfo directoryInfo, ExtendedGa4GhApi extendedGa4GhApi, boolean skipDockstore) { @@ -84,61 +86,57 @@ private void aggregateMetricsForDirectory(S3DirectoryInfo directoryInfo, Extende String toolId = directoryInfo.toolId(); String versionName = directoryInfo.versionId(); List platforms = directoryInfo.platforms(); - String platformsString = String.join(", ", platforms); String versionS3KeyPrefix = directoryInfo.versionS3KeyPrefix(); // Collect metrics for each platform, so we can calculate metrics across all platforms - List allMetrics = new ArrayList<>(); + Map platformToMetrics = new HashMap<>(); for (String platform : platforms) { ExecutionsRequestBody allSubmissions; try { allSubmissions = getExecutions(toolId, versionName, platform); } catch (Exception e) { LOG.error("Error aggregating metrics: Could not get all executions from directory {}", versionS3KeyPrefix, e); - numberOfMetricsSkipped.incrementAndGet(); + numberOfVersionsSkipped.incrementAndGet(); continue; // Continue aggregating metrics for other directories } try { Optional aggregatedPlatformMetric = getAggregatedMetrics(allSubmissions); if (aggregatedPlatformMetric.isPresent()) { - if (!skipDockstore) { - extendedGa4GhApi.aggregatedMetricsPut(aggregatedPlatformMetric.get(), platform, toolId, versionName); - } LOG.info("Aggregated metrics for tool ID {}, version {}, platform {} from directory {}", toolId, versionName, platform, versionS3KeyPrefix); - allMetrics.add(aggregatedPlatformMetric.get()); - numberOfMetricsSubmitted.incrementAndGet(); + platformToMetrics.put(platform, aggregatedPlatformMetric.get()); } else { LOG.error("Error aggregating metrics for tool ID {}, version {}, platform {} from directory {}", toolId, versionName, platform, versionS3KeyPrefix); } } catch (Exception e) { LOG.error("Error aggregating metrics: Could not put all executions from directory {}", versionS3KeyPrefix, e); - numberOfMetricsSkipped.incrementAndGet(); + numberOfVersionsSkipped.incrementAndGet(); // Continue aggregating metrics for other platforms } } - if (!allMetrics.isEmpty()) { + if (!platformToMetrics.isEmpty()) { // Calculate metrics across all platforms by aggregating the aggregated metrics from each platform try { - getAggregatedMetrics(allMetrics).ifPresent(metrics -> { + getAggregatedMetrics(platformToMetrics.values().stream().toList()).ifPresent(metrics -> { + platformToMetrics.put(Partner.ALL.name(), metrics); if (!skipDockstore) { - extendedGa4GhApi.aggregatedMetricsPut(metrics, Partner.ALL.name(), toolId, versionName); + extendedGa4GhApi.aggregatedMetricsPut(platformToMetrics, toolId, versionName); } LOG.info("Aggregated metrics across all platforms ({}) for tool ID {}, version {} from directory {}", - platformsString, toolId, versionName, versionS3KeyPrefix); - allMetrics.add(metrics); - numberOfMetricsSubmitted.incrementAndGet(); + platformToMetrics.keySet(), toolId, versionName, versionS3KeyPrefix); + + numberOfVersionsSubmitted.incrementAndGet(); }); } catch (Exception e) { - LOG.error("Error aggregating metrics across all platforms ({}) for tool ID {}, version {} from directory {}", platformsString, toolId, versionName, versionS3KeyPrefix, e); - numberOfMetricsSkipped.incrementAndGet(); + LOG.error("Error aggregating metrics across all platforms ({}) for tool ID {}, version {} from directory {}", platformToMetrics.keySet(), toolId, versionName, versionS3KeyPrefix, e); + numberOfVersionsSkipped.incrementAndGet(); // Continue aggregating metrics for other directories } } else { LOG.error("Error aggregating metrics for directory {}: no platform metrics aggregated", versionS3KeyPrefix); - numberOfMetricsSkipped.incrementAndGet(); + numberOfVersionsSkipped.incrementAndGet(); } numberOfDirectoriesProcessed.incrementAndGet(); LOG.info("Processed {} directories", numberOfDirectoriesProcessed); @@ -292,6 +290,12 @@ public List getDirectoriesForTrsId(String trsId) { return getDirectories(s3KeyPrefix); } + public List getDirectoriesForTrsIdVersion(String trsId, String versionName) { + final String s3KeyPrefix = S3ClientHelper.convertToolIdToPartialKey(trsId) + "/" + versionName; + LOG.info("Getting directories for TRS ID {} and version {} with S3 key prefix {}", trsId, versionName, s3KeyPrefix); + return getDirectories(s3KeyPrefix); + } + public record S3DirectoryInfo(String toolId, String versionId, List platforms, String versionS3KeyPrefix, AthenaTablePartition athenaTablePartition) { } } diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java index d2884d7a..93018528 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java @@ -35,6 +35,7 @@ import io.dockstore.metricsaggregator.client.cli.CommandLineArgs.SubmitValidationData; import io.dockstore.openapi.client.ApiClient; import io.dockstore.openapi.client.api.ExtendedGa4GhApi; +import io.dockstore.openapi.client.model.EntryLiteAndVersionName; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.ValidationExecution; import io.dockstore.openapi.client.model.ValidationExecution.ValidatorToolEnum; @@ -165,8 +166,12 @@ private void aggregateMetrics(AggregateMetricsCommand aggregateMetricsCommand, M final Instant getDirectoriesStartTime = Instant.now(); List s3DirectoriesToAggregate; if (trsIdsToAggregate == null || trsIdsToAggregate.isEmpty()) { - LOG.info("Aggregating metrics for all entries"); - s3DirectoriesToAggregate = metricsAggregatorS3Client.getDirectories(); // Aggregate all directories + LOG.info("Aggregating metrics for all entries that have new executions to aggregate"); + List entryVersionsToAggregate = extendedGa4GhApi.getEntryVersionsToAggregate(); + s3DirectoriesToAggregate = entryVersionsToAggregate.stream() + .map(entryVersion -> metricsAggregatorS3Client.getDirectoriesForTrsIdVersion(entryVersion.getEntryLite().getTrsId(), entryVersion.getVersionName())) + .flatMap(Collection::stream) + .toList(); } else { LOG.info("Aggregating metrics for TRS IDs: {}", trsIdsToAggregate); s3DirectoriesToAggregate = trsIdsToAggregate.stream() diff --git a/pom.xml b/pom.xml index 094553cd..8f2ca77b 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ scm:git:git@github.com:dockstore/dockstore-support.git UTF-8 - 1.16.0-alpha.13 + 1.16.0-alpha.16 3.0.0-M5 2.22.2 false