Do some JIRA/GitHub version syncing

jira_automation/README.md

@@ -3,23 +3,27 @@ There are three applications in here to facilitate our JIRA/GitHub interaction
 1. io.dockstore.jira.MilestoneChecker - generates GitHub and JQL queries to find mismatches in the JIRA fix version and GitHub milestone. The JIRA fix version is
    a multi-value field; the GitHub milestone is a single-value field, so Unito doesn't sync them. We have to remember to manually
    keep them in sync; this program identifies cases we've missed.
-2. SprintStart - a barely started work in progress to automatically generate review tickets at the beginning of a sprint, which
+2. io.dockstore.jira.MilestoneResolver - Updates JIRA and GitHub
+3. SprintStart - a barely started work in progress to automatically generate review tickets at the beginning of a sprint, which
    is currently a manual and tedious process.
-3. io.dockstore.jira.ResolutionChecker - used to help find issues open in GitHub that are closed in JIRA. This was to diagnose an issue where 
+4. io.dockstore.jira.ResolutionChecker - used to help find issues open in GitHub that are closed in JIRA. This was to diagnose an issue where 
 Unito was seemingly mysteriously closing JIRA issues at random. It turned out to be because we hadn't properly configured
 a GitHub and JIRA user in Unito -- it's the Unito intended behavior. We currently don't need to run this, although if we have
 a configuration issue again, it could be useful in the future.
 
 # Auth
 
-* ResolutionChecker and MilestoneCheck require environment variable `GITHUB_TOKEN` be set to a GitHub personal access token.
-* SprintStart requires the environment variable `JIRA_TOKEN` be set to a JIRA token.
+* ResolutionChecker, MilestoneChecker, and MilestoneResolver require the environment variable `GITHUB_TOKEN` be set to a GitHub personal access token that has access to dockstore GitHub issues
+* SprintStart and MilestoneResolver require the environment variable `JIRA_TOKEN` be set to a JIRA token.
 
 # Usage
 
 I usually run in IntelliJ with a Run Configuration
 
 1. In Run Configuration, set the main class to io.dockstore.jira.MilestoneChecker or io.dockstore.jira.ResolutionChecker
 2. Add the environment variable `GITHUB_TOKEN` to your GitHub token.
-3. The console will print out generated queries, which you then paste into your browser.
+3. For MilestoneResolver, you also need to set these environment variables:
+    * `JIRA_USERNAME` to your JIRA user, e.g., jdoe@ucsc.edu
+    * `JIRA_TOKEN` to your JIRA token
+3. The console will print out generated urls, which you then paste into your browser.
 

jira_automation/pom.xml

@@ -38,6 +38,10 @@
       <artifactId>gson</artifactId>
       <version>2.10.1</version>
     </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
 
   </dependencies>
 
@@ -53,6 +57,41 @@
           <forceJavacCompilerUse>true</forceJavacCompilerUse>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>milestoneResolverId</id>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                  <manifestEntries>
+                    <Main-Class>io.dockstore.jira.MilestoneResolver</Main-Class>
+                  </manifestEntries>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+        <configuration>
+          <filters>
+            <filter>
+              <artifact>*:*</artifact>
+              <excludes>
+                <exclude>META-INF/*.SF</exclude>
+                <exclude>META-INF/*.DSA</exclude>
+                <exclude>META-INF/*.RSA</exclude>
+              </excludes>
+            </filter>
+          </filters>
+        </configuration>
+      </plugin>
+
     </plugins>
 
   </build>

jira_automation/src/main/java/io/dockstore/jira/JiraIssue.java

@@ -0,0 +1,19 @@
+package io.dockstore.jira;
+
+import java.util.Date;
+
+/**
+ * JIRA Rest API models. There is no OpenAPI definition nor Java library I could find that worked;
+ * this models the parts of the entities that we consume, e.g., <code>JiraIssue</code> has many
+ * more properties than the record has here, but we only need to access the ones in the record.
+ * @param key
+ * @param fields
+ */
+public record JiraIssue(String key, Fields fields) { }
+
+record Fields(Date updated, FixVersion[] fixVersions) { }
+
+record FixVersion(String name) { }
+
+record UpdateJiraIssue(UpdateFields fields) { }
+record UpdateFields(FixVersion[] fixVersions) { }

jira_automation/src/main/java/io/dockstore/jira/MilestoneChecker.java

@@ -64,7 +64,7 @@ private static String generateGitHubIssuesUrl(final List<JiraAndGithub> issues)
     }
 
     private static boolean milestoneAndFixVersionEqual(String jiraFixVersion, String milestone) {
-        return "Open-ended research tasks".equals(jiraFixVersion) && "Open ended research tasks".equals(milestone)
+        return Utils.JIRA_OPEN_ENDED_RESEARCH_TASKS.equals(jiraFixVersion) && Utils.GITHUB_OPEN_ENDED_RESEARCH_TASKS.equals(milestone)
             || Objects.equals(jiraFixVersion, milestone);
     }
 

jira_automation/src/main/java/io/dockstore/jira/MilestoneResolver.java

@@ -0,0 +1,179 @@
+package io.dockstore.jira;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.kohsuke.github.GHIssue;
+import org.kohsuke.github.GHMilestone;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Attempts to resolve JIRA DOCK ticket fix version mismatches with the corresponding GitHub milestone.
+ * Handles these cases:
+ *
+ * <ol>
+ *     <li>There is a JIRA fix version but no GitHub milestone -- sets the GitHub milestone to the JIRA fix version</li>
+ *     <li>There is a GitHub milestone but no JIRA fix version -- sets the JIRA fix version to the GitHub milestone</li>
+ * </ol>
+ *
+ * <p>It does not handle the case of the JIRA fix version not matching the GitHub milestone. It does
+ * print out a message saying the mismatch needs to be resolved manually. To resolve this automatically,
+ * the program would need to:</p>
+ *
+ * <ol>
+ *     <li>Figure out the timestamp of when the JIRA fix version was last set</li>
+ *     <li>Figure out the timestamp of when the GitHub milestone was set</li>
+ *     <li>Resolve the difference by using the most recently changed.</li>
+ * </ol>
+ *
+ * <p>This is possible, but didn't seem worth the extra work at this point. See
+ * https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issues/#api-rest-api-3-issue-issueidorkey-changelog-get to get
+ * the JIRA change log. Note that it is paginated so the invoker would have to account for that.</p>
+ *
+ */
+public final class MilestoneResolver {
+
+    /**
+     *  Pattern for finding a substring like "Fix Versions: Dockstore 1.15" in GitHub issue body, to extract
+     *  the "1.15"
+     */
+    private static final Pattern FIX_VERSIONS = Pattern.compile("((Fix Versions)|(fixVersions))(:\\s*(Dockstore )?(.*))");
+    private static final int FIX_VERSION_REG_EX_GROUP = 6;
+
+    private static final Logger LOG = LoggerFactory.getLogger(MilestoneResolver.class);
+
+    private MilestoneResolver() { }
+
+    public static void main(String[] args) throws IOException {
+        final List<JiraAndGithub> mismatchedIssues = findMismatchedIssues();
+        if (mismatchedIssues.isEmpty()) {
+            LOG.info("The JIRA fix version and GitHub milestone are in sync for all DOCK issues");
+        } else {
+            LOG.info("The following issues are mismatched:");
+            mismatchedIssues.forEach(MilestoneResolver::printMismatchedIssue);
+        }
+        mismatchedIssues.forEach(issue -> {
+            final GHIssue gitHubIssue = issue.ghIssue;
+            try {
+                final JiraIssue jiraIssue = Utils.getJiraIssue(issue.jiraIssueId);
+                final GHMilestone ghMilestone = gitHubIssue.getMilestone();
+                final String jiraIssueUrl = getJiraIssueUrl(jiraIssue.key());
+                LOG.info("Processing JIRA issue {}", jiraIssueUrl);
+                final FixVersion[] fixVersions = jiraIssue.fields().fixVersions();
+                if (fixVersions.length == 0) {
+                    // There is GitHub milestone but no JIRA fix version
+                    updateJiraIssue(jiraIssue.key(), ghMilestone.getTitle());
+                } else if (fixVersions.length == 1 && ghMilestone == null) {
+                    // There's a JIRA fix version, but no GitHub milestone, set the GitHub milestone
+                    updateGitHubMilestone(gitHubIssue.getNumber(), fixVersions[0].name());
+                } else {
+                    LOG.info("The fix version and milestone mismatch must be resolved manually for: {}}", getJiraIssueUrl(issue.jiraIssueId));
+                }
+            } catch (URISyntaxException | IOException | InterruptedException e) {
+                LOG.error("Error resolving %s".formatted(issue), e);
+            }
+        });
+    }
+
+    private static void updateGitHubMilestone(int gitHubIssue, String jiraFixVersion) {
+        if (Utils.updateGitHubMilestone(gitHubIssue, jiraFixVersion)) {
+            LOG.info("Updated GitHub milestone in {} to {}", gitHubIssue, jiraFixVersion);
+        } else {
+            LOG.error("Failed to update GitHub milestone in {}", gitHubIssue);
+        }
+    }
+
+    private static void updateJiraIssue(String jiraIssue, String gitHubMilestone)
+        throws URISyntaxException, IOException, InterruptedException {
+        final String jiraIssueUrl = getJiraIssueUrl(jiraIssue);
+        if (Utils.updateJiraFixVersion(jiraIssue, gitHubMilestone)) {
+            LOG.info("Updated fix version in {} to {}", jiraIssueUrl, gitHubMilestone);
+        } else {
+            LOG.error("Failed to update fix version in {}", jiraIssueUrl);
+        }
+    }
+
+    private static void printMismatchedIssue(final MilestoneResolver.JiraAndGithub issue) {
+        final GHIssue ghIssue = issue.ghIssue;
+        final GHMilestone ghMilestone = ghIssue.getMilestone();
+        final String notSet = "<not set>";
+        final String milestone = ghMilestone != null ? ghMilestone.getTitle() : notSet;
+        final String jiraFixVersion = findJiraFixVersion(ghIssue.getBody()).orElse(notSet);
+        LOG.info(
+            "GitHub {}, milestone {}; JIRA {}, fix version {}",
+                ghIssue.getNumber(),
+                milestone,
+                issue.jiraIssueId,
+                jiraFixVersion);
+    }
+
+    /**
+     * Finds issues where the GitHub milestone does not match the JIRA fix version
+     * @return
+     * @throws IOException
+     */
+    private static List<MilestoneResolver.JiraAndGithub> findMismatchedIssues() throws IOException {
+        final List<GHIssue> openIssues = Utils.findOpenIssues(Utils.getDockstoreRepository());
+        return openIssues.stream()
+            .filter(MilestoneResolver::milestoneAndFixVersionMismatch)
+            .map(ghIssue -> {
+                // If empty, Unito hasn't synced, JIRA issue does not yet exist
+                return Utils.findJiraIssueInBody(ghIssue)
+                    .map(jiraIssue -> new MilestoneResolver.JiraAndGithub(jiraIssue, ghIssue)).orElse(null);
+            })
+            .filter(Objects::nonNull)
+            .collect(Collectors.toList());
+    }
+
+    private static boolean milestoneAndFixVersionMismatch(final GHIssue ghIssue) {
+        final GHMilestone milestone = ghIssue.getMilestone();
+        final String body = ghIssue.getBody();
+        final Optional<String> jiraFixVersion = findJiraFixVersion(body);
+        if (jiraFixVersion.isPresent()) {
+            if (milestone == null) {
+                // There's a fix version in JIRA, but none in GitHub
+                return true;
+            }
+            return !milestoneAndFixVersionEqual(jiraFixVersion.get(), milestone.getTitle());
+        } else { // No fix version in JIRA, is there one in GitHub?
+            return milestone != null;
+        }
+    }
+
+    private static String generateJiraIssuesUrl(final List<MilestoneResolver.JiraAndGithub> issues) {
+        return "https://ucsc-cgl.atlassian.net/issues/?jql=project=DOCK AND "
+            + issues.stream().map(issue -> "key=\"" + issue.jiraIssueId() + "\"")
+            .collect(Collectors.joining(" or "));
+    }
+
+    private static String getJiraIssueUrl(String issueNumber) {
+        return "https://ucsc-cgl.atlassian.net/browse/%s".formatted(issueNumber);
+    }
+
+    private static String generateGitHubIssuesUrl(final List<MilestoneResolver.JiraAndGithub> issues) {
+        return "https://github.com/dockstore/dockstore/issues?q="
+            + issues.stream().map(issue -> "" + issue.ghIssue.getNumber())
+            .collect(Collectors.joining("+"));
+    }
+
+    private static boolean milestoneAndFixVersionEqual(String jiraFixVersion, String milestone) {
+        return Utils.JIRA_OPEN_ENDED_RESEARCH_TASKS.equals(jiraFixVersion) && Utils.GITHUB_OPEN_ENDED_RESEARCH_TASKS.equals(milestone)
+            || Objects.equals(jiraFixVersion, milestone);
+    }
+
+    private static Optional<String> findJiraFixVersion(String gitHubIssueBody) {
+        final Matcher matcher = FIX_VERSIONS.matcher(gitHubIssueBody);
+        if (matcher.find()) {
+            return Optional.of(matcher.group(FIX_VERSION_REG_EX_GROUP));
+        }
+        return Optional.empty();
+    }
+
+    record JiraAndGithub(String jiraIssueId, GHIssue ghIssue) { }
+}

jira_automation/src/main/java/io/dockstore/jira/SprintStart.java

@@ -16,7 +16,7 @@
 public final class SprintStart {
 
     private static final String PROJECT = "SEAB";
-    private static final String BASE_URL = "https://ucsc-cgl.atlassian.net/rest/api/3/";
+    private static final String BASE_URL = Utils.JIRA_REST_BASE_URL;
     private static final String USERS_URL = BASE_URL + "users?maxResults=500";
     private static final String PROJECT_URL = BASE_URL + "project/" + PROJECT;