Merge pull request #24 from CleverTap/find-missing-tests-on-rerun

Do not skip tests on rerun if maven run exited prematurely or was forcibly killed

pom.xml

@@ -7,7 +7,7 @@
     <groupId>com.clevertap</groupId>
     <artifactId>supertest-maven-plugin</artifactId>
     <packaging>maven-plugin</packaging>
-    <version>1.11</version>
+    <version>1.12</version>
     <description>A wrapper for Maven's Surefire Plugin, with advanced re-run capabilities.
     </description>
     <name>supertest-maven-plugin</name>
@@ -43,6 +43,11 @@
             <version>2.2.1</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>2.22.1</version>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>

src/main/java/com/clevertap/maven/plugins/supertest/RunResult.java

@@ -5,20 +5,19 @@
 
 public class RunResult {
 
-    final String className;
-    final List<String> testCases;
+    private final String className;
+    private final List<String> failedTestCases = new ArrayList<>();
 
     public RunResult(String className) {
         this.className = className;
-        this.testCases = new ArrayList<>();
     }
 
-    public void addTestCase(final String testCase) {
-        testCases.add(testCase);
+    public void addFailedTestCase(final String testCase) {
+        failedTestCases.add(testCase);
     }
 
-    public List<String> getTestCases() {
-        return testCases;
+    public List<String> getFailedTestCases() {
+        return failedTestCases;
     }
 
     public String getClassName() {

src/main/java/com/clevertap/maven/plugins/supertest/SuperTestMavenPlugin.java

@@ -8,18 +8,23 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import javax.xml.parsers.ParserConfigurationException;
 import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
 import org.apache.maven.plugins.annotations.Mojo;
 import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.project.MavenProject;
@@ -30,6 +35,8 @@ public class SuperTestMavenPlugin extends AbstractMojo {
     // this is the max time to wait in seconds for process termination after the stdout read is
     // finished or terminated
     private static final int STDOUT_POST_READ_WAIT_TIMEOUT = 10;
+    private static final String TEST_REGEX = "-Dtest=(.*?)(\\s|$)";
+    private static final Pattern TEST_REGEX_PATTERN = Pattern.compile(TEST_REGEX);
 
     private ExecutorService pool;
 
@@ -49,8 +56,13 @@ public class SuperTestMavenPlugin extends AbstractMojo {
     @Parameter(property = "shellNoActivityTimeout", readonly = true, defaultValue = "300")
     Integer shellNoActivityTimeout;
 
-    public void execute() throws MojoExecutionException {
+    @Parameter(property = "includes" )
+    List<String> includes;
 
+    @Parameter(property = "excludes" )
+    List<String> excludes;
+
+    public void execute() throws MojoExecutionException, MojoFailureException {
         if (mvnTestOpts == null) {
             mvnTestOpts = "";
         }
@@ -69,6 +81,14 @@ public void execute() throws MojoExecutionException {
         final String groupId = project.getGroupId();
 
         pool = Executors.newFixedThreadPool(1);
+        String testClassesDir = project.getBuild().getTestOutputDirectory();
+
+        Set<String> allTestClasses = new HashSet<>(
+                new TestListResolver(
+                        includes, excludes, getTest(), testClassesDir).scanDirectories());
+
+        getLog().info("Test classes dir: " + testClassesDir);
+        getLog().debug("Test classes found: " + String.join(",", allTestClasses));
 
         int exitCode;
         final String command = "mvn test " + buildProcessedMvnTestOpts(artifactId, groupId);
@@ -83,7 +103,7 @@ public void execute() throws MojoExecutionException {
         }
 
         // Strip -Dtest=... from the Maven opts if specified, since these were valid for the very first run only.
-        mvnTestOpts = mvnTestOpts.replaceAll("-Dtest=(.*?)(\\s|$)", "");
+        mvnTestOpts = mvnTestOpts.replaceAll(TEST_REGEX, "");
 
         for (int retryRunNumber = 1; retryRunNumber <= retryRunCount; retryRunNumber++) {
             final File[] xmlFileList = getXmlFileList(baseDir);
@@ -97,10 +117,11 @@ public void execute() throws MojoExecutionException {
                     throw new MojoExecutionException(
                             "Failed to parse surefire report! file=" + file, e);
                 }
-                classnameToTestcaseList.put(runResult.getClassName(), runResult.getTestCases());
+                classnameToTestcaseList.put(
+                        runResult.getClassName(), runResult.getFailedTestCases());
             }
 
-            final String runCommand = createRerunCommand(classnameToTestcaseList);
+            final String runCommand = createRerunCommand(allTestClasses, classnameToTestcaseList);
 
             // previous run exited with code > 0, but all tests were actually run successfully
             if (runCommand == null) {
@@ -134,6 +155,15 @@ public void execute() throws MojoExecutionException {
         }
     }
 
+    public String getTest() {
+        if (mvnTestOpts == null) {
+            return "";
+        }
+
+        Matcher matcher = TEST_REGEX_PATTERN.matcher(mvnTestOpts);
+        return matcher.find() ? matcher.group(1) : "";
+    }
+
     private StringBuilder buildProcessedMvnTestOpts(String artifactId, String groupId) {
         final StringBuilder processedMvnTestOpts = new StringBuilder(" ");
         processedMvnTestOpts.append(mvnTestOpts);
@@ -251,31 +281,58 @@ public File[] getXmlFileList(File baseDir) {
      * @param classnameToTestcaseList map of classname and list of failed test cases
      * @return rerunCommand
      */
-    public String createRerunCommand(Map<String, List<String>> classnameToTestcaseList) {
-        boolean hasTestsAppended = false;
+    public String createRerunCommand(
+            Set<String> allTestClasses, Map<String, List<String>> classnameToTestcaseList) {
         final StringBuilder retryRun = new StringBuilder("mvn test");
+        Set<String> incompleteTests = new HashSet<>(allTestClasses);
+
         retryRun.append(" -Dtest=");
+        int emptyRetryRunLen = retryRun.length();
+
         // TODO: 04/02/2022 replace with Java 8 streams
         for (String className : classnameToTestcaseList.keySet()) {
+            // if a test class is in the surefire report, it means that all its tests were executed
+            incompleteTests.remove(className);
             List<String> failedTestCaseList = classnameToTestcaseList.get(className);
+
             if (!failedTestCaseList.isEmpty()) {
-                retryRun.append(className);
-                hasTestsAppended = true;
-                if(failedTestCaseList.contains("")) {
-                    retryRun.append(",");
-                    continue;
-                }
-                retryRun.append("#");
-                for (int i = 0; i < failedTestCaseList.size(); i++) {
-                    retryRun.append(failedTestCaseList.get(i)).append("*");
-                    if (i == failedTestCaseList.size() - 1) {
-                        retryRun.append(",");
-                    } else {
-                        retryRun.append("+");
-                    }
-                }
+                appendFailedTestCases(className, failedTestCaseList, retryRun);
+            } else {
+                // passing tests will not be re-run anymore
+                allTestClasses.remove(className);
+            }
+        }
+
+        if (retryRun.length() != emptyRetryRunLen
+                && retryRun.charAt(retryRun.length() - 1) != ','
+                && !incompleteTests.isEmpty()) {
+            retryRun.append(",");
+        }
+
+        retryRun.append(String.join(",", incompleteTests));
+
+        return retryRun.length() != emptyRetryRunLen ? retryRun.toString() : null;
+    }
+
+    private void appendFailedTestCases(
+            String className, List<String> failedTestCaseList, StringBuilder retryRun) {
+        retryRun.append(className);
+
+        if (failedTestCaseList.contains("")) {
+            retryRun.append(",");
+            return;
+        }
+
+        retryRun.append("#");
+
+        for (int i = 0; i < failedTestCaseList.size(); i++) {
+            retryRun.append(failedTestCaseList.get(i)).append("*");
+
+            if (i == failedTestCaseList.size() - 1) {
+                retryRun.append(",");
+            } else {
+                retryRun.append("+");
             }
         }
-        return hasTestsAppended ? retryRun.toString() : null;
     }
 }

src/main/java/com/clevertap/maven/plugins/supertest/SurefireReportParser.java

@@ -50,7 +50,7 @@ public RunResult parse() throws ParserConfigurationException, IOException, SAXEx
                 uniqueNames.add(name);
             }
         }
-        uniqueNames.forEach(result::addTestCase);
+        uniqueNames.forEach(result::addFailedTestCase);
         return result;
     }
 

src/main/java/com/clevertap/maven/plugins/supertest/TestListResolver.java

@@ -0,0 +1,84 @@
+package com.clevertap.maven.plugins.supertest;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.maven.plugin.MojoFailureException;
+import org.apache.maven.plugin.surefire.util.DirectoryScanner;
+
+public class TestListResolver {
+    private static final String[] DEFAULT_INCLUDES = new String[] {
+            "**/Test*.java", "**/*Test.java", "**/*Tests.java", "**/*TestCase.java"};
+    private static final String[] DEFAULT_EXCLUDES = new String[] {"**/*$*"};
+
+    private final DirectoryScanner scanner;
+
+    public TestListResolver(
+            List<String> includes, List<String> excludes, String test, String testClassDir)
+            throws MojoFailureException {
+        scanner = new DirectoryScanner(
+                new File(testClassDir),
+                new org.apache.maven.surefire.testset.TestListResolver(
+                        getIncludeList(test, includes), getExcludeList(test, excludes)));
+    }
+
+    public List<String> scanDirectories() {
+        return scanner.scan().getClasses();
+    }
+
+    private List<String> getIncludeList(String test, List<String> includes)
+            throws MojoFailureException {
+        return getFilterList(test, includes, DEFAULT_INCLUDES, x -> x.split(","));
+    }
+
+    private List<String> getExcludeList(String test, List<String> excludes)
+            throws MojoFailureException {
+        return getFilterList(test, excludes, DEFAULT_EXCLUDES, x -> new String[] {});
+    }
+
+    private List<String> getFilterList(
+            String test,
+            List<String> filterData,
+            String[] defaultFilterData,
+            Function<String, String[]> parseTestFunc) throws MojoFailureException {
+        List<String> filterList = new ArrayList<>();
+
+        if (isSpecificTestSpecified(test)) {
+            Collections.addAll(filterList, parseTestFunc.apply(test));
+        } else {
+            if (filterData != null) {
+                filterList.addAll(filterData);
+                checkMethodFilterInIncludesExcludes(filterList);
+            }
+
+            if (filterList.isEmpty()) {
+                Collections.addAll(filterList, defaultFilterData);
+            }
+        }
+
+        return filterNulls(filterList);
+    }
+
+    private static boolean isSpecificTestSpecified(String test) {
+        return test != null && !test.isEmpty();
+    }
+
+    private static void checkMethodFilterInIncludesExcludes(Iterable<String> patterns)
+            throws MojoFailureException {
+        for (String pattern : patterns) {
+            if (pattern != null && pattern.contains( "#" )) {
+                throw new MojoFailureException(
+                        "Method filter prohibited in includes|excludes parameter: " + pattern);
+            }
+        }
+    }
+
+    private static List<String> filterNulls(List<String> toFilter) {
+        return toFilter.stream()
+                .filter(x -> x != null && !x.trim().isEmpty())
+                .collect(Collectors.toList());
+    }
+}