mark classes that have test case but no test plan annotation as ignored

testah-junit/build.gradle

@@ -3,7 +3,6 @@ import org.gradle.api.internal.ConventionTask
 buildscript {
     repositories {
         mavenCentral()
-        jcenter()
         maven {
             url "https://plugins.gradle.org/m2/"
         }
@@ -35,7 +34,7 @@ def getEnvirVariable = {
         }
         return value
     } else {
-        System.out.println("Missing env MAVEN_REPO_TO_USE. Using /tmp instead");
+        System.out.println("Missing env MAVEN_REPO_TO_USE. Using /tmp instead")
         value = "file:///tmp"
     }
 }
@@ -61,7 +60,6 @@ repositories {
     //    url "http://repo1.maven.org/maven2"
     //}
     mavenCentral()
-    jcenter()
 }
 
 // Set checkstyle version.
@@ -121,7 +119,7 @@ dependencies {
     compile group: 'org.apache.poi', name: 'poi-ooxml', version: '4.1.2'
     compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
 
-    testCompile "com.github.tomakehurst:wiremock-standalone:2.27.2"
+    compile "com.github.tomakehurst:wiremock-standalone:2.27.2"
 
     // https://mvnrepository.com/artifact/org.glassfish.jersey.media/jersey-media-json-jackson
     compile group: 'org.glassfish.jersey.media', name: 'jersey-media-json-jackson', version: '2.26'

testah-junit/gradle.properties

@@ -1,4 +1,4 @@
 group=org.testah
-version=3.4.10
+version=3.4.11
 junitPlatformVersion=5.7.0
 log4JVersion=2.17.1

testah-junit/src/main/java/org/testah/framework/cli/Cli.java

@@ -10,7 +10,13 @@
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.extension.ExtensionConfigurationException;
+import org.junit.jupiter.api.extension.ExtensionContextException;
+import org.junit.jupiter.api.extension.TestInstantiationException;
+import org.junit.jupiter.params.aggregator.ArgumentAccessException;
+import org.junit.jupiter.params.aggregator.ArgumentsAggregationException;
 import org.junit.jupiter.params.provider.CsvParsingException;
+import org.junit.platform.commons.JUnitException;
 import org.junit.platform.commons.PreconditionViolationException;
 import org.junit.platform.launcher.listeners.TestExecutionSummary;
 import org.testah.TS;
@@ -52,7 +58,7 @@ public class Cli {
     /**
      * The Constant version.
      */
-    public static final String version = "3.4.10";
+    public static final String version = "3.4.11";
 
     /**
      * The Constant BAR_LONG.
@@ -444,7 +450,13 @@ public List<TestExecutionSummary.Failure> getInitializationErrors(ResultDto resu
                     .stream()
                     .filter(failure ->
                         failure.getException() instanceof PreconditionViolationException ||
-                            failure.getException() instanceof CsvParsingException
+                        failure.getException() instanceof ArgumentsAggregationException ||
+                        failure.getException() instanceof CsvParsingException ||
+                        failure.getException() instanceof ExtensionContextException ||
+                        failure.getException() instanceof TestInstantiationException ||
+                        failure.getException() instanceof ArgumentAccessException ||
+                        failure.getException() instanceof ExtensionConfigurationException ||
+                        failure.getException().getClass().equals(JUnitException.class)
                     )
                     .collect(Collectors.toList());
                 initFailures.forEach(failure -> initializationErrorFailures.add(failure.getException().getMessage()));

testah-junit/src/main/java/org/testah/framework/cli/TestFilter.java

@@ -5,6 +5,7 @@
 import org.apache.cxf.helpers.FileUtils;
 import org.codehaus.groovy.control.CompilationFailedException;
 import org.reflections.Reflections;
+import org.reflections.scanners.MethodAnnotationsScanner;
 import org.testah.TS;
 import org.testah.client.dto.TestCaseDto;
 import org.testah.client.enums.TestType;
@@ -15,13 +16,16 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase;
 import static org.apache.commons.lang3.StringUtils.trimToEmpty;
@@ -72,11 +76,57 @@ public int loadCompiledTestClass(final String internalClass) {
             Set<Class<?>> lst = reflections.getTypesAnnotatedWith(TestPlan.class);
             lst.addAll(reflections.getTypesAnnotatedWith(TestPlanJUnit5.class));
             testClasses.addAll(lst);
+            recordTestClassesWithoutTestPlansTestCases(
+                internalClass,
+                org.testah.framework.annotations.TestPlanJUnit5.class,
+                org.testah.framework.annotations.TestCaseJUnit5.class,
+                org.testah.framework.annotations.TestCaseWithParamsJUnit5.class
+            );
+            recordTestClassesWithoutTestPlansTestCases(
+                internalClass,
+                org.testah.framework.annotations.TestPlan.class,
+                org.testah.framework.annotations.TestCase.class
+            );
             return lst.size();
         }
         return 0;
     }
 
+    /**
+     * Record all classes that have Testah test case annotation but no Testah test plan annotation.
+     *
+     * @param internalClass base test package path
+     * @param testPlanAnnotation annotation marking the class as test plan
+     * @param testCaseAnnotations annotations marking methods as test methods
+     * @param <T> test plan annotation type
+     */
+    @SafeVarargs
+    public final <T extends Annotation> void recordTestClassesWithoutTestPlansTestCases(
+        final String internalClass,
+        final Class<T> testPlanAnnotation,
+        final Class<? extends Annotation>... testCaseAnnotations) {
+        final Reflections reflectionsCase = new Reflections(internalClass, new MethodAnnotationsScanner());
+
+        Set<Class<?>> testClassesWithoutTestPlanAnnotation = new HashSet<>();
+        for (Class<? extends Annotation> testCaseAnnotation : testCaseAnnotations) {
+            Set<Method> methods = reflectionsCase.getMethodsAnnotatedWith(testCaseAnnotation);
+            Set<Class<?>> declaringClasses =  methods.stream().map(Method::getDeclaringClass).collect(Collectors.toSet());
+            Set<Class<?>> filteredClasses = declaringClasses.stream().filter(it ->
+                    it.getAnnotation(testPlanAnnotation) == null
+            ).collect(Collectors.toSet());
+            testClassesWithoutTestPlanAnnotation.addAll(filteredClasses);
+        }
+        testClassesWithoutTestPlanAnnotation.forEach(clazz ->
+        {
+            long testCaseCount = Arrays.stream(testCaseAnnotations).map(testCaseAnnotation ->
+                new Reflections(clazz.getCanonicalName(), new MethodAnnotationsScanner())
+                    .getMethodsAnnotatedWith(testCaseAnnotation).size()).count();
+            TS.log().warn(String.format("Class %s has %d methods with test case annotations, but no test plan annotation.",
+                clazz.getCanonicalName(), testCaseCount));
+            recordIgnoredTestPlan(clazz.getCanonicalName(), (int) testCaseCount);
+        });
+    }
+
     /**
      * Filter test plans to run.
      *

testah-junit/src/test/java/org/testah/Junit5CsvDataProviderInvalidType.java

@@ -0,0 +1,23 @@
+package org.testah;
+
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+import org.junit.jupiter.params.provider.CsvFileSource;
+import org.testah.client.enums.TestType;
+import org.testah.framework.annotations.TestCaseWithParamsJUnit5;
+import org.testah.framework.annotations.TestPlanJUnit5;
+
+import static org.testah.framework.cli.CliTest.OVERRIDE_JUNIT5_CSV_INVALID_TYPE;
+
+@EnabledIfSystemProperty(named = OVERRIDE_JUNIT5_CSV_INVALID_TYPE, matches = "true")
+@TestPlanJUnit5(name = "test plan for junit5 test with data provider",
+    tags = OVERRIDE_JUNIT5_CSV_INVALID_TYPE, testType = TestType.AUTOMATED)
+public class Junit5CsvDataProviderInvalidType
+{
+
+    @TestCaseWithParamsJUnit5
+    @CsvFileSource(resources = "/org/testah/data.csv", numLinesToSkip = 1)
+    public void testCsvDataProviderInvalidType(String name, Integer testId) {
+        TS.asserts().equalsTo("", "testname", name);
+        TS.asserts().equalsTo("", "testId", testId);
+    }
+}

testah-junit/src/test/java/org/testah/Junit5CsvDataProviderNoSuchColumn.java

@@ -0,0 +1,23 @@
+package org.testah;
+
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+import org.junit.jupiter.params.provider.CsvFileSource;
+import org.testah.client.enums.TestType;
+import org.testah.framework.annotations.TestCaseWithParamsJUnit5;
+import org.testah.framework.annotations.TestPlanJUnit5;
+
+import static org.testah.framework.cli.CliTest.OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN;
+
+@EnabledIfSystemProperty(named = OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN, matches = "true")
+@TestPlanJUnit5(name = "test plan for junit5 test with data provider",
+    tags = OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN, testType = TestType.AUTOMATED)
+public class Junit5CsvDataProviderNoSuchColumn
+{
+    @TestCaseWithParamsJUnit5
+    @CsvFileSource(resources = "/org/testah/data.csv", numLinesToSkip = 1)
+    public void testCsvDataProvider(String name, String testId, String noSuchColumn) {
+        TS.asserts().equalsTo("", "testname", name);
+        TS.asserts().equalsTo("", "testId", testId);
+        TS.asserts().equalsTo("", "noSuchColumn", noSuchColumn);
+    }
+}

testah-junit/src/test/java/org/testah/Junit5TestPlanNegative.java → testah-junit/src/test/java/org/testah/Junit5CsvDataProviderNoSuchFile.java

@@ -1,18 +1,21 @@
 package org.testah;
 
-import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
 import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 import org.junit.jupiter.params.provider.CsvFileSource;
+import org.testah.client.enums.TestType;
 import org.testah.framework.annotations.TestCaseWithParamsJUnit5;
 import org.testah.framework.annotations.TestPlanJUnit5;
 
-@EnabledIfSystemProperty(named = "OVERRIDE_JUNIT5", matches = "true")
-@TestPlanJUnit5(name = "test plan for junit5 test with data provider", tags = "JUNIT_5_NEG")
-public class Junit5TestPlanNegative
+import static org.testah.framework.cli.CliTest.OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE;
+
+@EnabledIfSystemProperty(named = OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE, matches = "true")
+@TestPlanJUnit5(name = "test plan for junit5 test with data provider",
+    tags = OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE, testType = TestType.AUTOMATED)
+public class Junit5CsvDataProviderNoSuchFile
 {
     @TestCaseWithParamsJUnit5
     @CsvFileSource(resources = "/org/testah/runner/invalid/path/data.csv", numLinesToSkip = 1)
-    public void testCsvDataProvider(String name) {
+    public void testCsvDataProviderInvalidFile(String name) {
         TS.asserts().equalsTo("", "testname", name);
     }
 }

testah-junit/src/test/java/org/testah/Junit5MethodDataProviderNoSuchMethod.java

@@ -0,0 +1,20 @@
+package org.testah;
+
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.testah.client.enums.TestType;
+import org.testah.framework.annotations.TestCaseWithParamsJUnit5;
+import org.testah.framework.annotations.TestPlanJUnit5;
+
+import static org.testah.framework.cli.CliTest.OVERRIDE_JUNIT5_NO_SUCH_METHOD;
+
+@EnabledIfSystemProperty(named = OVERRIDE_JUNIT5_NO_SUCH_METHOD, matches = "true")
+@TestPlanJUnit5(name = "test plan for junit5 test with data provider", tags = OVERRIDE_JUNIT5_NO_SUCH_METHOD, testType = TestType.AUTOMATED)
+public class Junit5MethodDataProviderNoSuchMethod
+{
+    @TestCaseWithParamsJUnit5
+    @MethodSource("noSuchSourceMethod")
+    public void testMethodDataProvider(String name) {
+        TS.asserts().equalsTo("", "testname", name);
+    }
+}

testah-junit/src/test/java/org/testah/TestCaseClassJUnit4.java

@@ -0,0 +1,12 @@
+package org.testah;
+
+
+import org.testah.framework.annotations.TestCase;
+
+public class TestCaseClassJUnit4
+{
+    @TestCase(name = "test for junit4 example")
+    public void testCaseWithoutTestPlan() {
+        TS.asserts().isTrue(true);
+    }
+}

testah-junit/src/test/java/org/testah/TestCaseClassJUnit5.java

@@ -0,0 +1,11 @@
+package org.testah;
+
+import org.testah.framework.annotations.TestCaseJUnit5;
+
+public class TestCaseClassJUnit5
+{
+    @TestCaseJUnit5(name = "test for junit5 example")
+    public void testCaseWithoutTestPlan() {
+        TS.asserts().isTrue(true);
+    }
+}

testah-junit/src/test/java/org/testah/formatter/TestFormatters.java

@@ -40,5 +40,4 @@ public void test2() {
     public void tearDown() {
         TS.browser().close();
     }
-
 }

testah-junit/src/test/java/org/testah/framework/cli/CliTest.java

@@ -14,7 +14,10 @@ public class CliTest {
     private static final String ORG_TESTAH = "org.testah";
     private static final String PARAM_LOOK_AT_INTERNAL_TESTS = "param_lookAtInternalTests";
     private static final String FILTER_BY_TAG = "filter_DEFAULT_filterByTag";
-    public static final String OVERRIDE_JUNIT_5 = "OVERRIDE_JUNIT5";
+    public static final String OVERRIDE_JUNIT5_CSV_INVALID_TYPE = "OVERRIDE_JUNIT5_CSV_NO_SUCH_TYPE";
+    public static final String OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN = "OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN";
+    public static final String OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE = "OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE";
+    public static final String OVERRIDE_JUNIT5_NO_SUCH_METHOD = "OVERRIDE_JUNIT5_METHOD_NO_SUCH_METHOD";
 
     @Before
     public void setup() {
@@ -149,22 +152,41 @@ public void testCliLongBarWithCenterTextEmpty() {
     }
 
     @Test
-    public void testInitializationError() {
+    public void testInitializationErrorNoCsvFile() {
+        runJUnitInitializationTest(OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE, "does not exist");
+    }
+
+    @Test
+    public void testInitializationErrorInvalidType() {
+        runJUnitInitializationTest(OVERRIDE_JUNIT5_CSV_INVALID_TYPE, "Failed to convert String");
+    }
+
+    @Test
+    public void testInitializationErrorNoSuchCsvColumn() {
+        runJUnitInitializationTest(OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN, "Junit5CsvDataProviderNoSuchColumn");
+    }
+
+    @Test
+    public void testInitializationErrorNoSuchMethod() {
+        runJUnitInitializationTest(OVERRIDE_JUNIT5_NO_SUCH_METHOD, "noSuchSourceMethod");
+    }
+
+    private void runJUnitInitializationTest(String tag, String errorString) {
         final String[] args = {"run"};
         boolean found = false;
-        System.setProperty(FILTER_BY_TAG, "JUNIT_5_NEG");
-        System.setProperty(OVERRIDE_JUNIT_5, "true");
+        System.setProperty(FILTER_BY_TAG, tag);
+        System.setProperty(tag, "true");
         Cli cli = new Cli();
         try {
             cli.getArgumentParser(args);
-        } catch (RuntimeException e) {
+        } catch (Throwable e) {
             found =
                 cli.getResults().stream().anyMatch(
-                    resultDto -> resultDto.getTestPlan().getTestCases().get(0).getDescription().contains("does not exist")
+                    resultDto -> resultDto.getTestPlan().getTestCases().get(0).getExceptions().contains(errorString)
                 );
         } finally {
             System.clearProperty(FILTER_BY_TAG);
-            System.clearProperty(OVERRIDE_JUNIT_5);
+            System.clearProperty(tag);
         }
         Assert.assertTrue(found);
     }

testah-junit/src/test/java/org/testah/runner/performance/TestLoadTest.java

@@ -2,6 +2,7 @@
 
 import com.github.tomakehurst.wiremock.WireMockServer;
 import org.apache.http.HttpStatus;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.testah.TS;
@@ -43,6 +44,15 @@ public void setup() throws IOException
             .withStatus(HttpStatus.SC_ACCEPTED).withFixedDelay(RESPONSE_TIME_MILLIS)));
     }
 
+    /**
+     * Need to stop the Wiremock server once the tests\ completed.
+     */
+    @After
+    public void teardown()
+    {
+        wm.stop();
+    }
+
     @TestCase
     @Test
     public void runLoadTest() throws Exception

testah-junit/src/test/resources/org/testah/data.csv

@@ -0,0 +1,2 @@
+name,testid
+testname,testdescription