Ingest broad metrics using apache csv

THIRD-PARTY-LICENSES.txt

@@ -131,10 +131,10 @@ Lists of 417 third-party dependencies.
      (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.0 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.0 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-transport-httpclient5 (com.github.docker-java:docker-java-transport-httpclient5:3.3.0 - https://github.com/docker-java/docker-java)
-     (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-SNAPSHOT - no url defined)
      (Apache License 2.0) Dropwizard (io.dropwizard:dropwizard-core:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-core)
      (Apache License 2.0) Dropwizard Asset Bundle (io.dropwizard:dropwizard-assets:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-assets)
      (Apache License 2.0) Dropwizard Authentication (io.dropwizard:dropwizard-auth:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-auth)
@@ -309,9 +309,9 @@ Lists of 417 third-party dependencies.
      (MIT License) liquibase-slf4j (com.mattbertolini:liquibase-slf4j:5.0.0 - https://github.com/mattbertolini/liquibase-slf4j)
      (Apache License 2.0) localstack-utils (cloud.localstack:localstack-utils:0.2.22 - http://localstack.cloud)
      (Apache Software Licenses) Log4j Implemented Over SLF4J (org.slf4j:log4j-over-slf4j:2.0.9 - http://www.slf4j.org)
-     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Access Module (ch.qos.logback:logback-access:1.4.11 - http://logback.qos.ch/logback-access)
-     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.4.11 - http://logback.qos.ch/logback-classic)
-     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.4.11 - http://logback.qos.ch/logback-core)
+     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Access Module (ch.qos.logback:logback-access:1.4.12 - http://logback.qos.ch/logback-access)
+     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.4.12 - http://logback.qos.ch/logback-classic)
+     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.4.12 - http://logback.qos.ch/logback-core)
      (Apache License, Version 2.0) (MIT License) Logstash Logback Encoder (net.logstash.logback:logstash-logback-encoder:4.11 - https://github.com/logstash/logstash-logback-encoder)
      (Apache License, Version 2.0) Lucene Core (org.apache.lucene:lucene-core:8.7.0 - https://lucene.apache.org/lucene-parent/lucene-core)
      (MIT) mbknor-jackson-jsonSchema (com.kjetland:mbknor-jackson-jsonschema_2.12:1.0.34 - https://github.com/mbknor/mbknor-jackson-jsonSchema)
@@ -354,7 +354,7 @@ Lists of 417 third-party dependencies.
      (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis)
      (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/)
      (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/)
-     (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-SNAPSHOT - no url defined)
      (The Apache License, Version 2.0) OpenCensus (io.opencensus:opencensus-api:0.31.0 - https://github.com/census-instrumentation/opencensus-java)
      (Apache 2) opencsv (com.opencsv:opencsv:5.7.1 - http://opencsv.sf.net)
      (Apache 2.0) optics (io.circe:circe-optics_2.13:0.14.1 - https://github.com/circe/circe-optics)
@@ -395,7 +395,7 @@ Lists of 417 third-party dependencies.
      (Apache License 2.0) swagger-core-jakarta (io.swagger.core.v3:swagger-core-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-core-jakarta)
      (Apache License 2.0) swagger-integration-jakarta (io.swagger.core.v3:swagger-integration-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-integration-jakarta)
      (Apache Software License, Version 2.0) swagger-java-bitbucket-client (io.dockstore:swagger-java-bitbucket-client:2.0.3 - no url defined)
-     (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-SNAPSHOT - no url defined)
      (Apache Software License, Version 2.0) swagger-java-discourse-client (io.dockstore:swagger-java-discourse-client:2.0.1 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-quay-client (io.dockstore:swagger-java-quay-client:2.0.2 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-sam-client (io.dockstore:swagger-java-sam-client:2.0.2 - no url defined)

metricsaggregator/pom.xml

@@ -132,6 +132,14 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>s3</artifactId>
@@ -144,6 +152,10 @@
             <groupId>javax.money</groupId>
             <artifactId>money-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
@@ -278,6 +290,8 @@
                                 <usedDependency>org.glassfish.jersey.inject:jersey-hk2</usedDependency>
                                 <usedDependency>javax.money:money-api</usedDependency>
                                 <usedDependency>org.javamoney.moneta:moneta-core</usedDependency>
+                                <usedDependency>ch.qos.logback:logback-classic</usedDependency>
+                                <usedDependency>ch.qos.logback:logback-core</usedDependency>
                             </usedDependencies>
                         </configuration>
                     </execution>

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/CommandLineArgs.java

@@ -42,7 +42,7 @@ public File getConfig() {
     }
 
     @Parameters(commandNames = { "submit-validation-data" }, commandDescription = "Formats workflow validation data specified in a file then submits it to Dockstore")
-    public static class SubmitValidationData extends  CommandLineArgs {
+    public static class SubmitValidationData extends CommandLineArgs {
         @Parameter(names = {"-c", "--config"}, description = "The config file path.")
         private File config = new File("./" + MetricsAggregatorClient.CONFIG_FILE_NAME);
 
@@ -78,4 +78,45 @@ public Partner getPlatform() {
             return platform;
         }
     }
+
+    @Parameters(commandNames = { "submit-terra-metrics" }, commandDescription = "Formats workflow validation data specified in a file then submits it to Dockstore")
+    public static class SubmitTerraMetrics extends CommandLineArgs {
+        @Parameter(names = {"-c", "--config"}, description = "The config file path.")
+        private File config = new File("./" + MetricsAggregatorClient.CONFIG_FILE_NAME);
+
+
+        @Parameter(names = {"-d", "--data"}, description = "The file path to the CSV file containing workflow metrics from Terra. The first line of the file should contain the CSV fields: workflow_id,status,workflow_start,workflow_end,workflow_runtime_minutes,source_url", required = true)
+        private String dataFilePath;
+
+        @Parameter(names = {"-r", "--recordSkipped"}, description = "Record skipped executions and the reason skipped to a CSV file")
+        private boolean recordSkippedExecutions;
+
+        public File getConfig() {
+            return config;
+        }
+
+
+        public String getDataFilePath() {
+            return dataFilePath;
+        }
+
+        public boolean isRecordSkippedExecutions() {
+            return recordSkippedExecutions;
+        }
+
+        /**
+         * Headers for the input data file
+         */
+        public enum TerraMetricsCsvHeaders {
+            workflow_id, status, workflow_start, workflow_end, workflow_runtime_minutes, source_url
+        }
+
+        /**
+         * Headers for the output file containing workflow executions that were skipped.
+         * The headers are the same as the input file headers, with the addition of a "reason" header indicating why an execution was skipped
+         */
+        public enum SkippedTerraMetricsCsvHeaders {
+            workflow_id, status, workflow_start, workflow_end, workflow_runtime_minutes, source_url, reason_skipped
+        }
+    }
 }

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java

@@ -19,6 +19,7 @@
 
 import static io.dockstore.utils.CLIConstants.FAILURE_EXIT_CODE;
 import static io.dockstore.utils.ConfigFileUtils.getConfiguration;
+import static io.dockstore.utils.DockstoreApiClientUtils.setupApiClient;
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.MissingCommandException;
@@ -27,9 +28,9 @@
 import io.dockstore.metricsaggregator.MetricsAggregatorConfig;
 import io.dockstore.metricsaggregator.MetricsAggregatorS3Client;
 import io.dockstore.metricsaggregator.client.cli.CommandLineArgs.AggregateMetricsCommand;
+import io.dockstore.metricsaggregator.client.cli.CommandLineArgs.SubmitTerraMetrics;
 import io.dockstore.metricsaggregator.client.cli.CommandLineArgs.SubmitValidationData;
 import io.dockstore.openapi.client.ApiClient;
-import io.dockstore.openapi.client.Configuration;
 import io.dockstore.openapi.client.api.ExtendedGa4GhApi;
 import io.dockstore.openapi.client.model.ExecutionsRequestBody;
 import io.dockstore.openapi.client.model.ValidationExecution;
@@ -38,6 +39,8 @@
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 import java.util.Optional;
 import org.apache.commons.configuration2.INIConfiguration;
@@ -56,19 +59,22 @@ public class MetricsAggregatorClient {
 
     private static final Logger LOG = LoggerFactory.getLogger(MetricsAggregatorClient.class);
 
-
     public MetricsAggregatorClient() {
 
     }
 
     public static void main(String[] args) {
+        final Instant startTime = Instant.now();
         MetricsAggregatorClient metricsAggregatorClient = new MetricsAggregatorClient();
         final CommandLineArgs commandLineArgs = new CommandLineArgs();
         final JCommander jCommander = new JCommander(commandLineArgs);
         final AggregateMetricsCommand aggregateMetricsCommand = new AggregateMetricsCommand();
         final SubmitValidationData submitValidationData = new SubmitValidationData();
+        final SubmitTerraMetrics submitTerraMetrics = new SubmitTerraMetrics();
+
         jCommander.addCommand(aggregateMetricsCommand);
         jCommander.addCommand(submitValidationData);
+        jCommander.addCommand(submitTerraMetrics);
 
         try {
             jCommander.parse(args);
@@ -117,20 +123,35 @@ public static void main(String[] args) {
                 try {
                     final MetricsAggregatorConfig metricsAggregatorConfig = new MetricsAggregatorConfig(config.get());
                     metricsAggregatorClient.submitValidationData(metricsAggregatorConfig, submitValidationData.getValidator(),
-                            submitValidationData.getValidatorVersion(), submitValidationData.getDataFilePath(), submitValidationData.getPlatform());
+                            submitValidationData.getValidatorVersion(), submitValidationData.getDataFilePath(),
+                            submitValidationData.getPlatform());
                 } catch (Exception e) {
                     LOG.error("Could not submit validation metrics to Dockstore", e);
                     System.exit(FAILURE_EXIT_CODE);
                 }
             }
-        }
-    }
+        } else if ("submit-terra-metrics".equals(jCommander.getParsedCommand())) {
+            if (submitTerraMetrics.isHelp()) {
+                jCommander.usage();
+            } else {
+                final Optional<INIConfiguration> config = getConfiguration(submitTerraMetrics.getConfig());
+                if (config.isEmpty()) {
+                    System.exit(FAILURE_EXIT_CODE);
+                }
 
-    private ApiClient setupApiClient(String serverUrl, String token) {
-        ApiClient apiClient = Configuration.getDefaultApiClient();
-        apiClient.setBasePath(serverUrl);
-        apiClient.addDefaultHeader("Authorization", "Bearer " + token);
-        return apiClient;
+                try {
+                    final MetricsAggregatorConfig metricsAggregatorConfig = new MetricsAggregatorConfig(config.get());
+                    final TerraMetricsSubmitter submitTerraMetricsCommand = new TerraMetricsSubmitter(metricsAggregatorConfig,
+                            submitTerraMetrics);
+                    submitTerraMetricsCommand.submitMetrics();
+                } catch (Exception e) {
+                    LOG.error("Could not submit terra metrics to Dockstore", e);
+                    System.exit(FAILURE_EXIT_CODE);
+                }
+            }
+        }
+        final Instant endTime = Instant.now();
+        LOG.info("{} took {}", jCommander.getParsedCommand(), Duration.between(startTime, endTime));
     }
 
     private void aggregateMetrics(MetricsAggregatorConfig config) throws URISyntaxException {
@@ -147,8 +168,8 @@ private void aggregateMetrics(MetricsAggregatorConfig config) throws URISyntaxEx
         metricsAggregatorS3Client.aggregateMetrics(extendedGa4GhApi);
     }
 
-
-    private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolEnum validator, String validatorVersion, String dataFilePath, Partner platform) throws IOException {
+    private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolEnum validator, String validatorVersion,
+            String dataFilePath, Partner platform) throws IOException {
         ApiClient apiClient = setupApiClient(config.getDockstoreServerUrl(), config.getDockstoreToken());
         ExtendedGa4GhApi extendedGa4GhApi = new ExtendedGa4GhApi(apiClient);
 
@@ -179,17 +200,16 @@ private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolE
                 continue;
             }
             String dateExecuted = lineComponents[DATE_EXECUTED_INDEX];
-            ValidationExecution validationExecution = new ValidationExecution()
-                    .validatorTool(validator)
-                    .validatorToolVersion(validatorVersion)
-                    .isValid(isValid);
+            ValidationExecution validationExecution = new ValidationExecution().validatorTool(validator)
+                    .validatorToolVersion(validatorVersion).isValid(isValid);
             validationExecution.setDateExecuted(dateExecuted);
             ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().validationExecutions(List.of(validationExecution));
 
             try {
                 extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform.toString(), trsId, versionName,
                         "Validation executions submitted using dockstore-support metricsaggregator");
-                System.out.printf("Submitted validation metrics for tool ID %s, version %s, %s validated by %s %s on platform %s%n", trsId, versionName, isValid ? "successfully" : "unsuccessfully", validator, validatorVersion, platform);
+                System.out.printf("Submitted validation metrics for tool ID %s, version %s, %s validated by %s %s on platform %s%n", trsId,
+                        versionName, isValid ? "successfully" : "unsuccessfully", validator, validatorVersion, platform);
             } catch (Exception e) {
                 // Could end up here if the workflow no longer exists. Log then continue processing
                 LOG.error("Could not submit validation executions to Dockstore for workflow {}", csvLine, e);