diff --git a/testah-junit/build.gradle b/testah-junit/build.gradle index 006999f..647d3ed 100755 --- a/testah-junit/build.gradle +++ b/testah-junit/build.gradle @@ -3,7 +3,6 @@ import org.gradle.api.internal.ConventionTask buildscript { repositories { mavenCentral() - jcenter() maven { url "https://plugins.gradle.org/m2/" } @@ -35,7 +34,7 @@ def getEnvirVariable = { } return value } else { - System.out.println("Missing env MAVEN_REPO_TO_USE. Using /tmp instead"); + System.out.println("Missing env MAVEN_REPO_TO_USE. Using /tmp instead") value = "file:///tmp" } } @@ -61,7 +60,6 @@ repositories { // url "http://repo1.maven.org/maven2" //} mavenCentral() - jcenter() } // Set checkstyle version. @@ -121,7 +119,7 @@ dependencies { compile group: 'org.apache.poi', name: 'poi-ooxml', version: '4.1.2' compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' - testCompile "com.github.tomakehurst:wiremock-standalone:2.27.2" + compile "com.github.tomakehurst:wiremock-standalone:2.27.2" // https://mvnrepository.com/artifact/org.glassfish.jersey.media/jersey-media-json-jackson compile group: 'org.glassfish.jersey.media', name: 'jersey-media-json-jackson', version: '2.26' diff --git a/testah-junit/gradle.properties b/testah-junit/gradle.properties index 43d0f20..09f9668 100644 --- a/testah-junit/gradle.properties +++ b/testah-junit/gradle.properties @@ -1,4 +1,4 @@ group=org.testah -version=3.4.10 +version=3.4.11 junitPlatformVersion=5.7.0 log4JVersion=2.17.1 diff --git a/testah-junit/src/main/java/org/testah/framework/cli/Cli.java b/testah-junit/src/main/java/org/testah/framework/cli/Cli.java index bad71b4..c8e143a 100755 --- a/testah-junit/src/main/java/org/testah/framework/cli/Cli.java +++ b/testah-junit/src/main/java/org/testah/framework/cli/Cli.java @@ -10,7 +10,13 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.extension.ExtensionConfigurationException; +import org.junit.jupiter.api.extension.ExtensionContextException; +import org.junit.jupiter.api.extension.TestInstantiationException; +import org.junit.jupiter.params.aggregator.ArgumentAccessException; +import org.junit.jupiter.params.aggregator.ArgumentsAggregationException; import org.junit.jupiter.params.provider.CsvParsingException; +import org.junit.platform.commons.JUnitException; import org.junit.platform.commons.PreconditionViolationException; import org.junit.platform.launcher.listeners.TestExecutionSummary; import org.testah.TS; @@ -52,7 +58,7 @@ public class Cli { /** * The Constant version. */ - public static final String version = "3.4.10"; + public static final String version = "3.4.11"; /** * The Constant BAR_LONG. @@ -444,7 +450,13 @@ public List getInitializationErrors(ResultDto resu .stream() .filter(failure -> failure.getException() instanceof PreconditionViolationException || - failure.getException() instanceof CsvParsingException + failure.getException() instanceof ArgumentsAggregationException || + failure.getException() instanceof CsvParsingException || + failure.getException() instanceof ExtensionContextException || + failure.getException() instanceof TestInstantiationException || + failure.getException() instanceof ArgumentAccessException || + failure.getException() instanceof ExtensionConfigurationException || + failure.getException().getClass().equals(JUnitException.class) ) .collect(Collectors.toList()); initFailures.forEach(failure -> initializationErrorFailures.add(failure.getException().getMessage())); diff --git a/testah-junit/src/main/java/org/testah/framework/cli/TestFilter.java b/testah-junit/src/main/java/org/testah/framework/cli/TestFilter.java index cf790f1..9d383ef 100755 --- a/testah-junit/src/main/java/org/testah/framework/cli/TestFilter.java +++ b/testah-junit/src/main/java/org/testah/framework/cli/TestFilter.java @@ -5,6 +5,7 @@ import org.apache.cxf.helpers.FileUtils; import org.codehaus.groovy.control.CompilationFailedException; import org.reflections.Reflections; +import org.reflections.scanners.MethodAnnotationsScanner; import org.testah.TS; import org.testah.client.dto.TestCaseDto; import org.testah.client.enums.TestType; @@ -15,6 +16,8 @@ import java.io.File; import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.ArrayList; @@ -22,6 +25,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase; import static org.apache.commons.lang3.StringUtils.trimToEmpty; @@ -72,11 +76,57 @@ public int loadCompiledTestClass(final String internalClass) { Set> lst = reflections.getTypesAnnotatedWith(TestPlan.class); lst.addAll(reflections.getTypesAnnotatedWith(TestPlanJUnit5.class)); testClasses.addAll(lst); + recordTestClassesWithoutTestPlansTestCases( + internalClass, + org.testah.framework.annotations.TestPlanJUnit5.class, + org.testah.framework.annotations.TestCaseJUnit5.class, + org.testah.framework.annotations.TestCaseWithParamsJUnit5.class + ); + recordTestClassesWithoutTestPlansTestCases( + internalClass, + org.testah.framework.annotations.TestPlan.class, + org.testah.framework.annotations.TestCase.class + ); return lst.size(); } return 0; } + /** + * Record all classes that have Testah test case annotation but no Testah test plan annotation. + * + * @param internalClass base test package path + * @param testPlanAnnotation annotation marking the class as test plan + * @param testCaseAnnotations annotations marking methods as test methods + * @param test plan annotation type + */ + @SafeVarargs + public final void recordTestClassesWithoutTestPlansTestCases( + final String internalClass, + final Class testPlanAnnotation, + final Class... testCaseAnnotations) { + final Reflections reflectionsCase = new Reflections(internalClass, new MethodAnnotationsScanner()); + + Set> testClassesWithoutTestPlanAnnotation = new HashSet<>(); + for (Class testCaseAnnotation : testCaseAnnotations) { + Set methods = reflectionsCase.getMethodsAnnotatedWith(testCaseAnnotation); + Set> declaringClasses = methods.stream().map(Method::getDeclaringClass).collect(Collectors.toSet()); + Set> filteredClasses = declaringClasses.stream().filter(it -> + it.getAnnotation(testPlanAnnotation) == null + ).collect(Collectors.toSet()); + testClassesWithoutTestPlanAnnotation.addAll(filteredClasses); + } + testClassesWithoutTestPlanAnnotation.forEach(clazz -> + { + long testCaseCount = Arrays.stream(testCaseAnnotations).map(testCaseAnnotation -> + new Reflections(clazz.getCanonicalName(), new MethodAnnotationsScanner()) + .getMethodsAnnotatedWith(testCaseAnnotation).size()).count(); + TS.log().warn(String.format("Class %s has %d methods with test case annotations, but no test plan annotation.", + clazz.getCanonicalName(), testCaseCount)); + recordIgnoredTestPlan(clazz.getCanonicalName(), (int) testCaseCount); + }); + } + /** * Filter test plans to run. * diff --git a/testah-junit/src/test/java/org/testah/Junit5CsvDataProviderInvalidType.java b/testah-junit/src/test/java/org/testah/Junit5CsvDataProviderInvalidType.java new file mode 100644 index 0000000..a5d7467 --- /dev/null +++ b/testah-junit/src/test/java/org/testah/Junit5CsvDataProviderInvalidType.java @@ -0,0 +1,23 @@ +package org.testah; + +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.params.provider.CsvFileSource; +import org.testah.client.enums.TestType; +import org.testah.framework.annotations.TestCaseWithParamsJUnit5; +import org.testah.framework.annotations.TestPlanJUnit5; + +import static org.testah.framework.cli.CliTest.OVERRIDE_JUNIT5_CSV_INVALID_TYPE; + +@EnabledIfSystemProperty(named = OVERRIDE_JUNIT5_CSV_INVALID_TYPE, matches = "true") +@TestPlanJUnit5(name = "test plan for junit5 test with data provider", + tags = OVERRIDE_JUNIT5_CSV_INVALID_TYPE, testType = TestType.AUTOMATED) +public class Junit5CsvDataProviderInvalidType +{ + + @TestCaseWithParamsJUnit5 + @CsvFileSource(resources = "/org/testah/data.csv", numLinesToSkip = 1) + public void testCsvDataProviderInvalidType(String name, Integer testId) { + TS.asserts().equalsTo("", "testname", name); + TS.asserts().equalsTo("", "testId", testId); + } +} diff --git a/testah-junit/src/test/java/org/testah/Junit5CsvDataProviderNoSuchColumn.java b/testah-junit/src/test/java/org/testah/Junit5CsvDataProviderNoSuchColumn.java new file mode 100644 index 0000000..afbe8c5 --- /dev/null +++ b/testah-junit/src/test/java/org/testah/Junit5CsvDataProviderNoSuchColumn.java @@ -0,0 +1,23 @@ +package org.testah; + +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.params.provider.CsvFileSource; +import org.testah.client.enums.TestType; +import org.testah.framework.annotations.TestCaseWithParamsJUnit5; +import org.testah.framework.annotations.TestPlanJUnit5; + +import static org.testah.framework.cli.CliTest.OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN; + +@EnabledIfSystemProperty(named = OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN, matches = "true") +@TestPlanJUnit5(name = "test plan for junit5 test with data provider", + tags = OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN, testType = TestType.AUTOMATED) +public class Junit5CsvDataProviderNoSuchColumn +{ + @TestCaseWithParamsJUnit5 + @CsvFileSource(resources = "/org/testah/data.csv", numLinesToSkip = 1) + public void testCsvDataProvider(String name, String testId, String noSuchColumn) { + TS.asserts().equalsTo("", "testname", name); + TS.asserts().equalsTo("", "testId", testId); + TS.asserts().equalsTo("", "noSuchColumn", noSuchColumn); + } +} diff --git a/testah-junit/src/test/java/org/testah/Junit5TestPlanNegative.java b/testah-junit/src/test/java/org/testah/Junit5CsvDataProviderNoSuchFile.java similarity index 56% rename from testah-junit/src/test/java/org/testah/Junit5TestPlanNegative.java rename to testah-junit/src/test/java/org/testah/Junit5CsvDataProviderNoSuchFile.java index cb9eb6e..f165250 100644 --- a/testah-junit/src/test/java/org/testah/Junit5TestPlanNegative.java +++ b/testah-junit/src/test/java/org/testah/Junit5CsvDataProviderNoSuchFile.java @@ -1,18 +1,21 @@ package org.testah; -import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.params.provider.CsvFileSource; +import org.testah.client.enums.TestType; import org.testah.framework.annotations.TestCaseWithParamsJUnit5; import org.testah.framework.annotations.TestPlanJUnit5; -@EnabledIfSystemProperty(named = "OVERRIDE_JUNIT5", matches = "true") -@TestPlanJUnit5(name = "test plan for junit5 test with data provider", tags = "JUNIT_5_NEG") -public class Junit5TestPlanNegative +import static org.testah.framework.cli.CliTest.OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE; + +@EnabledIfSystemProperty(named = OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE, matches = "true") +@TestPlanJUnit5(name = "test plan for junit5 test with data provider", + tags = OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE, testType = TestType.AUTOMATED) +public class Junit5CsvDataProviderNoSuchFile { @TestCaseWithParamsJUnit5 @CsvFileSource(resources = "/org/testah/runner/invalid/path/data.csv", numLinesToSkip = 1) - public void testCsvDataProvider(String name) { + public void testCsvDataProviderInvalidFile(String name) { TS.asserts().equalsTo("", "testname", name); } } diff --git a/testah-junit/src/test/java/org/testah/Junit5MethodDataProviderNoSuchMethod.java b/testah-junit/src/test/java/org/testah/Junit5MethodDataProviderNoSuchMethod.java new file mode 100644 index 0000000..67cbae7 --- /dev/null +++ b/testah-junit/src/test/java/org/testah/Junit5MethodDataProviderNoSuchMethod.java @@ -0,0 +1,20 @@ +package org.testah; + +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.params.provider.MethodSource; +import org.testah.client.enums.TestType; +import org.testah.framework.annotations.TestCaseWithParamsJUnit5; +import org.testah.framework.annotations.TestPlanJUnit5; + +import static org.testah.framework.cli.CliTest.OVERRIDE_JUNIT5_NO_SUCH_METHOD; + +@EnabledIfSystemProperty(named = OVERRIDE_JUNIT5_NO_SUCH_METHOD, matches = "true") +@TestPlanJUnit5(name = "test plan for junit5 test with data provider", tags = OVERRIDE_JUNIT5_NO_SUCH_METHOD, testType = TestType.AUTOMATED) +public class Junit5MethodDataProviderNoSuchMethod +{ + @TestCaseWithParamsJUnit5 + @MethodSource("noSuchSourceMethod") + public void testMethodDataProvider(String name) { + TS.asserts().equalsTo("", "testname", name); + } +} diff --git a/testah-junit/src/test/java/org/testah/TestCaseClassJUnit4.java b/testah-junit/src/test/java/org/testah/TestCaseClassJUnit4.java new file mode 100644 index 0000000..ac6757c --- /dev/null +++ b/testah-junit/src/test/java/org/testah/TestCaseClassJUnit4.java @@ -0,0 +1,12 @@ +package org.testah; + + +import org.testah.framework.annotations.TestCase; + +public class TestCaseClassJUnit4 +{ + @TestCase(name = "test for junit4 example") + public void testCaseWithoutTestPlan() { + TS.asserts().isTrue(true); + } +} diff --git a/testah-junit/src/test/java/org/testah/TestCaseClassJUnit5.java b/testah-junit/src/test/java/org/testah/TestCaseClassJUnit5.java new file mode 100644 index 0000000..d8dad3c --- /dev/null +++ b/testah-junit/src/test/java/org/testah/TestCaseClassJUnit5.java @@ -0,0 +1,11 @@ +package org.testah; + +import org.testah.framework.annotations.TestCaseJUnit5; + +public class TestCaseClassJUnit5 +{ + @TestCaseJUnit5(name = "test for junit5 example") + public void testCaseWithoutTestPlan() { + TS.asserts().isTrue(true); + } +} diff --git a/testah-junit/src/test/java/org/testah/formatter/TestFormatters.java b/testah-junit/src/test/java/org/testah/formatter/TestFormatters.java index 1634d82..8ba6762 100755 --- a/testah-junit/src/test/java/org/testah/formatter/TestFormatters.java +++ b/testah-junit/src/test/java/org/testah/formatter/TestFormatters.java @@ -40,5 +40,4 @@ public void test2() { public void tearDown() { TS.browser().close(); } - } diff --git a/testah-junit/src/test/java/org/testah/framework/cli/CliTest.java b/testah-junit/src/test/java/org/testah/framework/cli/CliTest.java index 75d8766..36cec40 100755 --- a/testah-junit/src/test/java/org/testah/framework/cli/CliTest.java +++ b/testah-junit/src/test/java/org/testah/framework/cli/CliTest.java @@ -14,7 +14,10 @@ public class CliTest { private static final String ORG_TESTAH = "org.testah"; private static final String PARAM_LOOK_AT_INTERNAL_TESTS = "param_lookAtInternalTests"; private static final String FILTER_BY_TAG = "filter_DEFAULT_filterByTag"; - public static final String OVERRIDE_JUNIT_5 = "OVERRIDE_JUNIT5"; + public static final String OVERRIDE_JUNIT5_CSV_INVALID_TYPE = "OVERRIDE_JUNIT5_CSV_NO_SUCH_TYPE"; + public static final String OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN = "OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN"; + public static final String OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE = "OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE"; + public static final String OVERRIDE_JUNIT5_NO_SUCH_METHOD = "OVERRIDE_JUNIT5_METHOD_NO_SUCH_METHOD"; @Before public void setup() { @@ -149,22 +152,41 @@ public void testCliLongBarWithCenterTextEmpty() { } @Test - public void testInitializationError() { + public void testInitializationErrorNoCsvFile() { + runJUnitInitializationTest(OVERRIDE_JUNIT5_CSV_NO_SUCH_FILE, "does not exist"); + } + + @Test + public void testInitializationErrorInvalidType() { + runJUnitInitializationTest(OVERRIDE_JUNIT5_CSV_INVALID_TYPE, "Failed to convert String"); + } + + @Test + public void testInitializationErrorNoSuchCsvColumn() { + runJUnitInitializationTest(OVERRIDE_JUNIT5_CSV_NO_SUCH_COLUMN, "Junit5CsvDataProviderNoSuchColumn"); + } + + @Test + public void testInitializationErrorNoSuchMethod() { + runJUnitInitializationTest(OVERRIDE_JUNIT5_NO_SUCH_METHOD, "noSuchSourceMethod"); + } + + private void runJUnitInitializationTest(String tag, String errorString) { final String[] args = {"run"}; boolean found = false; - System.setProperty(FILTER_BY_TAG, "JUNIT_5_NEG"); - System.setProperty(OVERRIDE_JUNIT_5, "true"); + System.setProperty(FILTER_BY_TAG, tag); + System.setProperty(tag, "true"); Cli cli = new Cli(); try { cli.getArgumentParser(args); - } catch (RuntimeException e) { + } catch (Throwable e) { found = cli.getResults().stream().anyMatch( - resultDto -> resultDto.getTestPlan().getTestCases().get(0).getDescription().contains("does not exist") + resultDto -> resultDto.getTestPlan().getTestCases().get(0).getExceptions().contains(errorString) ); } finally { System.clearProperty(FILTER_BY_TAG); - System.clearProperty(OVERRIDE_JUNIT_5); + System.clearProperty(tag); } Assert.assertTrue(found); } diff --git a/testah-junit/src/test/java/org/testah/runner/performance/TestLoadTest.java b/testah-junit/src/test/java/org/testah/runner/performance/TestLoadTest.java index ea22ac9..c1248ab 100644 --- a/testah-junit/src/test/java/org/testah/runner/performance/TestLoadTest.java +++ b/testah-junit/src/test/java/org/testah/runner/performance/TestLoadTest.java @@ -2,6 +2,7 @@ import com.github.tomakehurst.wiremock.WireMockServer; import org.apache.http.HttpStatus; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.testah.TS; @@ -43,6 +44,15 @@ public void setup() throws IOException .withStatus(HttpStatus.SC_ACCEPTED).withFixedDelay(RESPONSE_TIME_MILLIS))); } + /** + * Need to stop the Wiremock server once the tests\ completed. + */ + @After + public void teardown() + { + wm.stop(); + } + @TestCase @Test public void runLoadTest() throws Exception diff --git a/testah-junit/src/test/resources/org/testah/data.csv b/testah-junit/src/test/resources/org/testah/data.csv new file mode 100644 index 0000000..c59792a --- /dev/null +++ b/testah-junit/src/test/resources/org/testah/data.csv @@ -0,0 +1,2 @@ +name,testid +testname,testdescription \ No newline at end of file